text: /*##############################################################################
  
      HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  
      Licensed under the Apache License, Version 2.0 (the "License");
      you may not use this file except in compliance with the License.
      You may obtain a copy of the License at
  
         http://www.apache.org/licenses/LICENSE-2.0
  
      Unless required by applicable law or agreed to in writing, software
      distributed under the License is distributed on an "AS IS" BASIS,
      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
      See the License for the specific language governing permissions and
      limitations under the License.
  ############################################################################## */
  
  #include "jmisc.hpp"
  #include "jdebug.hpp"
  #include "jptree.hpp"
  #include "rtlkey.hpp"
  #include "jsort.hpp"
  #include "jhtree.hpp"
  #include "jqueue.tpp"
  #include "jisem.hpp"
  #include "thorxmlread.hpp"
  #include "thorrparse.ipp"
  #include "thorxmlwrite.hpp"
  #include "thorsoapcall.hpp"
  #include "thorcommon.ipp"
  #include "thorsort.hpp"
  #include "thorstats.hpp"
  #include "jlzw.hpp"
  #include "javahash.hpp"
  #include "javahash.tpp"
  #include "thorstep.ipp"
  #include "thorpipe.hpp"
  #include "thorfile.hpp"
  #include "eclhelper.hpp"
  #include "eclrtl_imp.hpp"
  #include "rtlfield.hpp"
  #include "rtlds_imp.hpp"
  #include "rtlread_imp.hpp"
  #include "rtlcommon.hpp"
  
  #include <algorithm>
  
  #include "dafdesc.hpp"
  #include "dautils.hpp"
  #include "ftbase.ipp"
  
  #include "eclhelper_base.hpp"
  
  #include "ccd.hpp"
  #include "ccdserver.hpp"
  #include "ccdcontext.hpp"
  #include "ccdactivities.hpp"
  #include "ccdquery.hpp"
  #include "ccdstate.hpp"
  #include "ccdqueue.ipp"
  #include "ccdsnmp.hpp"
  #include "ccddali.hpp"
  #include "jsmartsock.hpp"
  
  #include "dllserver.hpp"
  #include "workflow.hpp"
  #include "roxiemem.hpp"
  #include "roxierowbuff.hpp"
  
  #include "roxiehelper.hpp"
  #include "roxielmj.hpp"
  #include "roxierow.hpp"
  
  #include "thorplugin.hpp"
  #include "keybuild.hpp"
  #include "thorstrand.hpp"
  #include "rtldynfield.hpp"
  
  #define MAX_HTTP_HEADERSIZE 8000
  #define MIN_PAYLOAD_SIZE 800
  
  #ifdef _WIN32
  #pragma warning(disable : 4355)
  #endif
  #define DEFAULT_PARALLEL_LOOP_THREADS       1
  
  #define PROBE
  #ifdef _DEBUG
  //#define FAKE_EXCEPTIONS
  //#define TRACE_JOINGROUPS
  //#define TRACE_SPLIT
  //#define _CHECK_HEAPSORT
  //#undef PARALLEL_EXECUTE
  //#define TRACE_SEEK_REQUESTS
  #endif
  
  using roxiemem::OwnedRoxieRow;
  using roxiemem::OwnedRoxieString;
  using roxiemem::OwnedConstRoxieRow;
  using roxiemem::IRowManager;
  using roxiemem::ReleaseRoxieRows;
  
  // There is a bug in VC6 implemetation of protected which prevents nested classes from accessing owner's data. It can be tricky to work around - hence...
  #if _MSC_VER==1200
  #define protected public
  #endif
  
  #define TRACE_STARTSTOP  // This determines if it is available - it is enabled/disabled by a configuration option
  #define TRACE_STRANDS
  #define MAX_SENSIBLE_STRANDS 1024 // Architecture dependent...
  
  static const SmartStepExtra dummySmartStepExtra(SSEFreadAhead, NULL);
  
  //=================================================================================
  
  class RestartableThread : public CInterface
  {
      class MyThread : public Thread
      {
          Linked<RestartableThread> owner;
      public:
          MyThread(RestartableThread *_owner, const char *name) : Thread(name), owner(_owner)
          {
          }
          virtual int run()
          {
              owner->started.signal();
              return owner->run();
          }
      };
      friend class MyThread;
      Semaphore started;
      Owned<MyThread> thread;
      CriticalSection crit;
      StringAttr name;
  public:
      RestartableThread(const char *_name) : name(_name)
      {
      }
      virtual void start(const char *namePrefix)
      {
          StringBuffer s(namePrefix);
          s.append(name);
          {
              CriticalBlock b(crit);
              assertex(!thread);
              thread.setown(new MyThread(this, s));
              thread->start();
          }
          started.wait();
      }
  
      virtual void join()
      {
          {
              Owned<Thread> tthread;
              {
                  CriticalBlock b(crit);
                  tthread.setown(thread.getClear());
              }
              if (tthread)
                  tthread->join();
          }
      }
  
      virtual int run() = 0;
  
  };
  
  //================================================================================
  
  //The following don't link their arguments because that creates a circular reference
  //But I wish there was a better way
  class IndirectAgentContext : implements IRoxieAgentContext, public CInterface
  {
  public:
      IndirectAgentContext(IRoxieAgentContext * _ctx) : ctx(_ctx) {}
      IMPLEMENT_IINTERFACE
  
  //    void set(IRoxieAgentContext * _ctx) { ctx = _ctx; }
  
      virtual ICodeContext *queryCodeContext()
      {
          return ctx->queryCodeContext();
      }
      virtual void checkAbort() 
      {
          ctx->checkAbort();
      }
      virtual unsigned checkInterval() const
      {
          return ctx->checkInterval();
      }
      virtual void notifyAbort(IException *E) 
      {
          ctx->notifyAbort(E);
      }
      virtual IActivityGraph * queryChildGraph(unsigned id) 
      {
          return ctx->queryChildGraph(id);
      }
      virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph) 
      {
          ctx->noteChildGraph(id, childGraph) ;
      }
      virtual IRowManager &queryRowManager() 
      {
          return ctx->queryRowManager();
      }
      virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const
      {
          ctx->noteStatistic(kind, value);
      }
      virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const
      {
          ctx->setStatistic(kind, value);
      }
      virtual void mergeStats(const CRuntimeStatisticCollection &from) const
      {
          ctx->mergeStats(from);
      }
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          ctx->gatherStats(merged);
      }
      virtual bool collectingDetailedStatistics() const
      {
          return ctx->collectingDetailedStatistics();
      }
      virtual void CTXLOGva(const char *format, va_list args) const __attribute__((format(printf,2,0)))
      {
          ctx->CTXLOGva(format, args);
      }
      virtual void CTXLOGa(TracingCategory category, const char *prefix, const char *text) const
      {
          ctx->CTXLOGa(category, prefix, text);
      }
      virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0)))
      {
          ctx->logOperatorExceptionVA(E, file, line, format, args);
      }
      virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute__((format(printf,6,0)))
      {
          ctx->CTXLOGaeva(E, file, line, prefix, format, args);
      }
      virtual void CTXLOGl(LogItem *log) const
      {
          ctx->CTXLOGl(log);
      }
      virtual StringBuffer &getLogPrefix(StringBuffer &ret) const override
      {
          return ctx->getLogPrefix(ret);
      }
      virtual unsigned queryTraceLevel() const
      {
          return ctx->queryTraceLevel();
      }
      virtual bool isIntercepted() const
      {
          return ctx->isIntercepted();
      }
      virtual bool isBlind() const
      {
          return ctx->isBlind();
      }
      virtual void setGlobalId(const char *id, SocketEndpoint &ep, unsigned pid) override
      {
          ctx->setGlobalId(id, ep, pid);
      }
      virtual void setCallerId(const char *id) override
      {
          ctx->setCallerId(id);
      }
      virtual const char *queryGlobalId() const
      {
          return ctx->queryGlobalId();
      }
      virtual const char *queryCallerId() const override
      {
          return ctx->queryCallerId();
      }
      virtual const char *queryLocalId() const
      {
          return ctx->queryLocalId();
      }
      virtual void setHttpIdHeaders(const char *global, const char *caller)
      {
          ctx->setHttpIdHeaders(global, caller);
      }
      virtual const char *queryGlobalIdHttpHeader() const
      {
          return ctx->queryGlobalIdHttpHeader();
      }
      virtual const char *queryCallerIdHttpHeader() const
      {
          return ctx->queryCallerIdHttpHeader();
      }
      virtual const QueryOptions &queryOptions() const
      {
          return ctx->queryOptions();
      }
      virtual void addAgentsReplyLen(unsigned len, unsigned duplicates, unsigned resends)
      {
          ctx->addAgentsReplyLen(len, duplicates, resends);
      }
      virtual const char *queryAuthToken() 
      {
          return ctx->queryAuthToken();
      }
      virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser)
      {
          return ctx->resolveLFN(filename, isOpt, isPrivilegedUser);
      }
      virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser)
      {
          return ctx->createLFN(filename, overwrite, extend, clusters, isPrivilegedUser);
      }
      virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser)
      {
          ctx->onFileCallback(header, lfn, isOpt, isLocal, isPrivilegedUser);
      }
      virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity)
      {
          return ctx->getLibraryGraph(extra, parentActivity);
      }
      virtual IProbeManager *queryProbeManager() const
      {
          return ctx->queryProbeManager();
      }
      virtual IDebuggableContext *queryDebugContext() const
      {
          return ctx->queryDebugContext();
      }
      virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence)
      {
          ctx->printResults(output, name, sequence);
      }
      virtual void setWUState(WUState state)
      {
          ctx->setWUState(state);
      }
      virtual bool checkWuAborted()
      {
          return ctx->checkWuAborted();
      }
      virtual IWorkUnit *updateWorkUnit() const
      {
          return ctx->updateWorkUnit();
      }
      virtual IConstWorkUnit *queryWorkUnit() const
      {
          return ctx->queryWorkUnit();
      }
      virtual IRoxieServerContext *queryServerContext()
      {
          return ctx->queryServerContext();
      }
      virtual IWorkUnitRowReader *getWorkunitRowReader(const char *wuid, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped)
      {
          return ctx->getWorkunitRowReader(wuid, name, sequence, xmlTransformer, rowAllocator, isGrouped);
      }
      virtual IEngineRowAllocator *getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, roxiemem::RoxieHeapFlags flags) const
      {
          return ctx->getRowAllocatorEx(meta, activityId, flags);
      }
  
  
  protected:
      IRoxieAgentContext * ctx;
  };
  
  //=================================================================================
  
  #define RESULT_FLUSH_THRESHOLD 10000u
  
  #ifdef _DEBUG
  #define SOAP_SPLIT_THRESHOLD 100u
  #define SOAP_SPLIT_RESERVE 200u
  #else
  #define SOAP_SPLIT_THRESHOLD 64000u
  #define SOAP_SPLIT_RESERVE 65535u
  #endif
  
  //=================================================================================
  
  // General activity statistics
  
  static const StatisticsMapping actStatistics({StWhenFirstRow, StTimeElapsed, StTimeLocalExecute, StTimeTotalExecute, StSizeMaxRowSize,
                                                StNumRowsProcessed, StNumSlaves, StNumStarts, StNumStops, StNumStrands,
                                                StNumScansPerRow, StNumAllocations, StNumAllocationScans,
                                                StTimeFirstExecute, StCycleLocalExecuteCycles, StCycleTotalExecuteCycles});
  static const StatisticsMapping joinStatistics({StNumAtmostTriggered}, actStatistics);
  static const StatisticsMapping keyedJoinStatistics({ StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks,
                                                      StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares,
                                                      StNumPreFiltered, StNumPostFiltered, StNumIndexAccepted, StNumIndexRejected,
                                                      StNumIndexRowsRead, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
                                                      StNumBlobCacheHits, StNumLeafCacheHits, StNumNodeCacheHits,
                                                      StNumBlobCacheAdds, StNumLeafCacheAdds, StNumNodeCacheAdds,
                                                      StNumDiskRejected, StSizeAgentReply, StTimeAgentWait}, joinStatistics);
  static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks,
                                                  StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares,
                                                  StNumPreFiltered, StNumPostFiltered, StNumIndexAccepted, StNumIndexRejected,
                                                  StNumBlobCacheHits, StNumLeafCacheHits, StNumNodeCacheHits,
                                                  StNumBlobCacheAdds, StNumLeafCacheAdds, StNumNodeCacheAdds,
                                                  StNumIndexRowsRead, StSizeAgentReply, StTimeAgentWait}, actStatistics);
  static const StatisticsMapping diskStatistics({StNumServerCacheHits, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
                                                 StNumDiskRejected, StSizeAgentReply, StTimeAgentWait }, actStatistics);
  static const StatisticsMapping soapStatistics({ StTimeSoapcall }, actStatistics);
  static const StatisticsMapping groupStatistics({ StNumGroups, StNumGroupMax }, actStatistics);
  static const StatisticsMapping sortStatistics({ StTimeSortElapsed }, actStatistics);
  static const StatisticsMapping indexWriteStatistics({ StNumDuplicateKeys }, actStatistics);
  
  // These ones get accumulated and reported in COMPLETE: line (and workunit).
  // Excludes ones that are not sensible to sum across activities, other than StTimeTotalExecute which must be explicitly overwritten at global level before we report it
  
  extern const StatisticsMapping accumulatedStatistics({StWhenFirstRow, StTimeLocalExecute, StTimeTotalExecute, StSizeMaxRowSize,
                                                        StNumRowsProcessed, StNumSlaves, StNumStarts, StNumStops, StNumStrands,
                                                        StNumScansPerRow, StNumAllocations, StNumAllocationScans,
                                                        StCycleLocalExecuteCycles,
                                                        StNumAtmostTriggered,
                                                        StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks,
                                                        StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares,
                                                        StNumPreFiltered, StNumPostFiltered, StNumIndexAccepted, StNumIndexRejected,
                                                        StNumIndexRowsRead, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
                                                        StNumBlobCacheHits, StNumLeafCacheHits, StNumNodeCacheHits,
                                                        StNumBlobCacheAdds, StNumLeafCacheAdds, StNumNodeCacheAdds,
                                                        StNumDiskRejected, StSizeAgentReply, StTimeAgentWait,
                                                        StTimeSoapcall,
                                                        StNumGroups,
                                                        StTimeSortElapsed,
                                                        StNumDuplicateKeys});
  
  //=================================================================================
  
  extern SinkMode getSinkMode(const char *val)
  {
      if (strieq(val, "parallelpersistent"))
          return SinkMode::ParallelPersistent;
      else if (strieq(val, "sequential"))
          return SinkMode::Sequential;
      else
      {
          if (!strieq(val, "parallel"))
              WARNLOG("Unsupported sinkmode %s - assuming parallel", val);
          return SinkMode::Parallel;
      }
  }
  
  
  const static unsigned minus1U = (0U-1U);
  class CRoxieServerActivityFactoryBase : public CActivityFactory, implements IRoxieServerActivityFactory
  {
  protected:
      IntArray dependencies; // things I am dependent on
      IntArray dependencyIndexes; // things I am dependent on
      IntArray dependencyControlIds; // things I am dependent on
      StringArray dependencyEdgeIds; // How to describe them to the debugger
      unsigned dependentCount; // things dependent on me
      unsigned optParallel = 0; // -1=enable,0=not specified,1=single-threaded,n=<num threads>
      bool optStableInput = true; // is the input forced to ordered?
      bool optUnstableInput = false;  // is the input forced to unordered?
      bool optUnordered = false; // is the output specified as unordered?
      bool isCodeSigned = false;
      SinkMode sinkMode = SinkMode::Parallel;
      unsigned heapFlags;
  
  public:
      IMPLEMENT_IINTERFACE;
  
      CRoxieServerActivityFactoryBase(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats)
          : CActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
          dependentCount = 0;
          optParallel = _graphNode.getPropInt("att[@name='parallel']/@value", 0);
          optUnordered = !_graphNode.getPropBool("att[@name='ordered']/@value", true);
          heapFlags = _graphNode.getPropInt("hint[@name='heapflags']/@value", _queryFactory.queryOptions().heapFlags);
          const char *sinkModeText  = _graphNode.queryProp("hint[@name='sinkmode']/@value");
          if (sinkModeText)
              sinkMode = ::getSinkMode(sinkModeText);
          else
              sinkMode = _queryFactory.queryOptions().sinkMode;
          isCodeSigned = ::isActivityCodeSigned(_graphNode);
      }
      
      ~CRoxieServerActivityFactoryBase()
      {
      }
  
      StringBuffer &toString(StringBuffer &ret) const
      {
          return ret.appendf("%p", this);
      }
  
      virtual void addDependency(unsigned _source, ThorActivityKind _kind, unsigned _sourceIdx, int controlId, const char *edgeId)
      {
          dependencies.append(_source); 
          dependencyIndexes.append(_sourceIdx); 
          dependencyControlIds.append(controlId);
          dependencyEdgeIds.append(edgeId);
      }
      virtual bool isInputOrdered(bool consumerOrdered, unsigned idx) const
      {
          if (optStableInput)
              return true;
          if (optUnstableInput)
              return false;
          if (optUnordered)
              return false;
          return consumerOrdered;
      }
  
      virtual void noteDependent(unsigned target)
      {
          dependentCount++;
      }
  
      virtual IntArray &queryDependencies() { return dependencies; } 
      virtual IntArray &queryDependencyIndexes() {    return dependencyIndexes; } 
      virtual IntArray &queryDependencyControlIds() { return dependencyControlIds; }
      virtual StringArray &queryDependencyEdgeIds() { return dependencyEdgeIds; }
      virtual unsigned queryId() const { return id; }
      virtual unsigned querySubgraphId() const { return subgraphId; }
      virtual ThorActivityKind getKind() const { return kind; }
      virtual IOutputMetaData * queryOutputMeta() const
      {
          return meta;
      }
      virtual bool isSink() const
      {
          return false;
      }
      virtual bool isFunction() const
      {
          return false;
      }
      virtual bool isGraphInvariant() const
      {
          return false;
      }
      virtual IHThorArg &getHelper() const
      {
          return *helperFactory();
      }
      virtual IRoxieServerActivity *createFunction(IHThorArg &arg, IProbeManager *_probeManager) const
      {
          arg.Release();
          throwUnexpected();
      }
  
      virtual void mergeStats(const CRuntimeStatisticCollection &from) const
      {
          CActivityFactory::mergeStats(from);
      }
  
      virtual void noteProcessed(unsigned _idx, unsigned _processed) const
      {
          dbgassertex(!_idx);
          if (likely(_processed))
              myedgestats.addStatistic(StNumRowsProcessed, _processed);
      }
  
      virtual void noteStarted() const
      {
          myedgestats.addStatistic(StNumStarts, 1);
          myedgestats.addStatistic(StNumStops, 1);    // Makes things prettier - we don't actually track stops at the factory level
          myedgestats.setStatistic(StNumSlaves, 1);   // Ditto - meaningless for Roxie but affects the display
      }
  
      virtual void noteStarted(unsigned idx) const
      {
          throwUnexpected(); // should be implemented/required by multiOutput cases only
      }
  
      virtual void resetNodeProgressInfo()
      {
          CActivityFactory::resetNodeProgressInfo();
      }
      virtual void getActivityMetrics(StringBuffer &reply) const
      {
          CActivityFactory::getActivityMetrics(reply);
      }
      virtual void gatherStats(IStatisticGatherer &builder, int channel, bool reset) const override
      {
          //Because subgraphs are flattened in roxie the subgraph needs to be selected for each activity
          dbgassertex(channel == -1);
          StatsSubgraphScope sg(builder, subgraphId);
          {
              StatsActivityScope ac(builder, id);
              mystats.recordStatistics(builder, reset);
              ForEachItemIn(i, childQueries)
              {
                  auto child = childQueries.item(i);
                  StatsChildGraphScope cc(builder, childQueryIndexes.item(i));
                  child.gatherStats(builder, channel, reset);
              }
          }
          gatherEdgeStats(builder, channel, reset);
      }
      virtual void gatherEdgeStats(IStatisticGatherer &builder, int channel, bool reset) const
      {
          StatsEdgeScope e(builder, id, 0);
          myedgestats.recordStatistics(builder, reset);
      }
      virtual unsigned __int64 queryLocalTimeNs() const
      {
          return mystats.getSerialStatisticValue(StTimeLocalExecute);
      }
      virtual IQueryFactory &queryQueryFactory() const
      {
          return CActivityFactory::queryQueryFactory();
      }
  
      virtual ActivityArray *queryChildQuery(unsigned idx, unsigned &id)
      {
          return CActivityFactory::queryChildQuery(idx, id);
      }
  
      virtual void addChildQuery(unsigned id, ActivityArray *childQuery)
      {
          CActivityFactory::addChildQuery(id, childQuery);
      }
  
      virtual void onCreateChildQueries(IRoxieAgentContext *ctx, IHThorArg *colocalArg, IArrayOf<IActivityGraph> &childGraphs, IRoxieServerActivity *parentActivity, IProbeManager *_probeManager, const IRoxieContextLogger &_logctx, unsigned numParallel) const
      {
          ForEachItemIn(idx, childQueries)
          {
              childGraphs.append(*createActivityGraph(ctx, NULL, childQueryIndexes.item(idx), childQueries.item(idx), parentActivity, _probeManager, _logctx, numParallel));
              ctx->noteChildGraph(childQueryIndexes.item(idx), &childGraphs.item(idx));
              childGraphs.item(idx).onCreate(colocalArg);
          }
      }
  
      IActivityGraph * createChildGraph(IRoxieAgentContext * ctx, IHThorArg *colocalArg, unsigned childId, IRoxieServerActivity *parentActivity, IProbeManager * _probeManager, const IRoxieContextLogger &_logctx) const
      {
          unsigned match = childQueryIndexes.find(childId);
          assertex(match != NotFound);
          Owned<IActivityGraph> graph = createActivityGraph(ctx, NULL, childQueryIndexes.item(match), childQueries.item(match), parentActivity, _probeManager, _logctx, 1);
          graph->onCreate(colocalArg);
          return graph.getClear();
      }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          // Most activities have nothing to say...
      }
      virtual const StatisticsMapping &queryStatsMapping() const
      {
          return mystats.queryMapping(); // Can be overridden by anyone that needs to collect different than factory
      }
      virtual roxiemem::RoxieHeapFlags getHeapFlags() const
      {
          return (roxiemem::RoxieHeapFlags)heapFlags;
      }
      virtual bool isActivityCodeSigned() const
      {
          return isCodeSigned;
      }
      virtual RecordTranslationMode getEnableFieldTranslation() const
      {
          return CActivityFactory::getEnableFieldTranslation();
      }
      virtual SinkMode getSinkMode() const override
      {
          return sinkMode;
      }
  };
  
  class CRoxieServerMultiInputInfo
  {
  private:
      UnsignedArray inputs;
      UnsignedArray inputIndexes;
  
  public:
      void set(unsigned idx, unsigned source, unsigned sourceidx)
      {
          if (idx==inputs.length())
          {
              inputs.append(source);
              inputIndexes.append(sourceidx);
          }
          else
          {
              while (!inputs.isItem(idx))
              {
                  inputs.append(0);
                  inputIndexes.append(0);
              }
              inputs.replace(source, idx);
              inputIndexes.replace(sourceidx, idx);
          }
      }
  
      unsigned get(unsigned idx, unsigned &sourceidx) const
      {
          if (inputs.isItem(idx))
          {
              sourceidx = inputIndexes.item(idx);
              return inputs.item(idx);
          }
          else
              return (unsigned) -1;
      }
  
      inline unsigned ordinality() const { return inputs.ordinality(); }
  };
  
  
  class CRoxieServerMultiInputFactory : public CRoxieServerActivityFactoryBase
  {
  private:
      CRoxieServerMultiInputInfo inputs;
  
  public:
      CRoxieServerMultiInputFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats)
          : CRoxieServerActivityFactoryBase(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          inputs.set(idx, source, sourceidx);
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          return inputs.get(idx, sourceidx);
      }
  
      virtual unsigned numInputs() const { return inputs.ordinality(); }
  };
  
  class DECL_EXCEPTION CWrappedException : implements IException, public CInterface
  {
      Owned<IException> wrapped;
      ThorActivityKind kind;
      unsigned queryId;
  
  public: 
      IMPLEMENT_IINTERFACE;
      CWrappedException(IException *_wrapped, ThorActivityKind _kind, unsigned _queryId) 
          : wrapped(_wrapped), kind(_kind), queryId(_queryId)
      {
      }
      virtual int             errorCode() const { return wrapped->errorCode(); }
      virtual StringBuffer &  errorMessage(StringBuffer &msg) const { return wrapped->errorMessage(msg).appendf(" (in %s %d)", getActivityText(kind), queryId); }
      virtual MessageAudience errorAudience() const { return wrapped->errorAudience(); }
  };
  
  class CRoxieServerActivityFactory : public CRoxieServerActivityFactoryBase
  {
  protected:
      unsigned input;
      unsigned inputidx;
  
  public:
      CRoxieServerActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats)
          : CRoxieServerActivityFactoryBase(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
          input = (unsigned) -1;
          inputidx = 0;
      }
  
      inline void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          if (idx != 0)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: id = %d : setInput() parameter out of bounds idx = %d at %s(%d)", id, idx, sanitizeSourceFile(__FILE__), __LINE__);
          if (input != (unsigned) -1)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: id = %d : setInput() called twice for input = %d source = %d  inputidx = %d  sourceidx = %d at %s(%d)", id, input, source, inputidx, sourceidx, sanitizeSourceFile(__FILE__), __LINE__);
          input = source;
          inputidx = sourceidx;
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          if (!idx)
          {
              sourceidx = inputidx;
              return input;
          }
          return (unsigned) -1;
      }
  
      virtual unsigned numInputs() const { return (input == (unsigned)-1) ? 0 : 1; }
  };
  
  static const StatisticsMapping edgeStatisticsX({StNumRowsProcessed, StNumStarts, StNumStops, StNumSlaves});
  
  class CRoxieServerMultiOutputFactory : public CRoxieServerActivityFactory
  {
  protected:
      unsigned numOutputs = 0;
      mutable std::vector<CRuntimeStatisticCollection> edgestatsArray;
  
      CRoxieServerMultiOutputFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
      }
  
      void setNumOutputs(unsigned num)
      {
          numOutputs = num;
          if (!num)
              num = 1; // Even sink activities like to track how many records they process
          edgestatsArray.reserve(num);
          for (unsigned i = 0; i < num; i++)
          {
              edgestatsArray.push_back(CRuntimeStatisticCollection(edgeStatisticsX));
          }
      }
  
      virtual void resetNodeProgressInfo()
      {
          CRoxieServerActivityFactory::resetNodeProgressInfo();
          for (unsigned i = 0; i < numOutputs; i++)
          {
              edgestatsArray[i].reset();
          }
      }
  
      virtual void gatherEdgeStats(IStatisticGatherer &builder, int channel, bool reset) const override
      {
          // Edges here (other than for splitter) are a bit odd - there may not be a corresponding edge in the graph to
          // record the stats against, and if there is the scope string for it is probably not what we expect, as these
          // edges are artificially added in response to tying spills to reads, etc.
          // So ignore them for now
      }
  
      virtual void noteProcessed(unsigned idx, unsigned _processed) const
       {
           assertex(numOutputs ? idx < numOutputs : idx==0);
           edgestatsArray[idx].addStatistic(StNumRowsProcessed, _processed);
       }
  
      virtual void noteStarted(unsigned idx) const
      {
          assertex(numOutputs ? idx < numOutputs : idx==0);
          edgestatsArray[idx].addStatistic(StNumStarts, 1);
          edgestatsArray[idx].addStatistic(StNumStops, 1);    // Makes things prettier - we don't actually track stops at the factory level
          edgestatsArray[idx].setStatistic(StNumSlaves, 1);   // Ditto - meaningless for Roxie but affects the display
      }
  };
  
  class CRoxieServerInternalSinkFactory : public CRoxieServerActivityFactory
  {
  protected:
      bool isInternal;
      bool isRoot;
      unsigned usageCount;
  public:
  
      CRoxieServerInternalSinkFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats, unsigned _usageCount, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
          usageCount = _usageCount;
          isRoot = _isRoot;
          isInternal = false; // filled in by derived class constructor
      }
  
      virtual bool isSink() const
      {
          //If an internal result has as many dependencies (within the graph) as uses (which includes from outside the graph) then don't execute it unconditionally.
          bool internalSpillAllUsesWithinGraph = (isInternal && dependentCount && dependentCount==usageCount);
          //only a sink if a root activity
          return isRoot && !internalSpillAllUsesWithinGraph;
      }
  
      virtual void gatherEdgeStats(IStatisticGatherer &builder, int channel, bool reset) const override
      {
          // There is no meaningful info to return along the dependency edge - we don't detect how many times the value has been read from the context
          // Just leave it blank is safest.
      }
  
  };
  
  typedef enum { STATEreset, STATEstarted, STATEstopped, STATEstarting } activityState;
  
  const char *queryStateText(activityState state)
  {
      switch (state)
      {
      case STATEreset: return "reset";
      case STATEstarted: return "started";
      case STATEstopped: return "stopped";
      case STATEstarting: return "starting";   // Used by a splitter to indicate it has seen a stop but not yet seen a start(), nor has it seen stop() on all output adaptors
      default: return "unknown";
      }
  }
  
  typedef ICopyArrayOf<IRoxieServerActivity> IRoxieServerActivityCopyArray;
  
  class CParallelActivityExecutor : public CAsyncFor
  {
  public:
      unsigned parentExtractSize;
      const byte * parentExtract;
  
      CParallelActivityExecutor(IRoxieServerActivityCopyArray & _activities, unsigned _parentExtractSize, const byte * _parentExtract) : 
          parentExtractSize(_parentExtractSize), parentExtract(_parentExtract), activities(_activities) { }
      void Do(unsigned i)
      {
          activities.item(i).execute(parentExtractSize, parentExtract);
      }
  private:
      IRoxieServerActivityCopyArray & activities;
  };
  
  extern IEngineRowStream * ensureSingleStream(IRoxieAgentContext *ctx, PointerArrayOf<IEngineRowStream> & instreams, Owned<IStrandJunction> &junction)
  {
      if (instreams.length() != 1)
      {
          assertex(instreams.length());
          if (!junction)
              junction.setown(createStrandJunction(ctx->queryRowManager(), instreams.length(), 1, ctx->queryOptions().strandBlockSize, false));
          ForEachItemIn(stream, instreams)
          {
              junction->setInput(stream, instreams.item(stream));
          }
          return junction->queryOutput(0);
      }
      else
          return instreams.item(0);
  }
  
  extern IEngineRowStream *connectSingleStream(IRoxieAgentContext *ctx, IFinalRoxieInput *input, unsigned idx, Owned<IStrandJunction> &junction, bool consumerOrdered)
  {
      if (input)
      {
          PointerArrayOf<IEngineRowStream> instreams;
          junction.setown(input->getOutputStreams(ctx, idx, instreams, NULL, consumerOrdered, nullptr));
          return ensureSingleStream(ctx, instreams, junction);
      }
      else
          return NULL;
  }
  
  extern IEngineRowStream *connectSingleStream(IRoxieAgentContext *ctx, IFinalRoxieInput *input, unsigned idx, bool consumerOrdered)
  {
      Owned<IStrandJunction> junction;
      IEngineRowStream * result = connectSingleStream(ctx, input, idx, junction, consumerOrdered);
      assertex(!junction);
      return result;
  }
  
  class CRoxieServerActivity : implements CInterfaceOf<IRoxieServerActivity>, implements IFinalRoxieInput, implements IEngineRowStream, implements IRoxieContextLogger
  {
      friend class StrandProcessor;
  protected:
      IFinalRoxieInput *input;
      unsigned sourceIdx;
      IEngineRowStream *inputStream;
      Owned<IStrandJunction> junction;
  
      IHThorArg &basehelper;
      IRoxieAgentContext *ctx;
      class InterceptRegisterTimer : public IndirectCodeContext
      {
      public:
          InterceptRegisterTimer(CRoxieServerActivity &_activity) : activity(_activity) {}
          virtual ISectionTimer *registerTimer(unsigned activityId, const char * name)
          {
              if (activityId == activity.queryId())
                  return activity.registerTimer(activityId, name);
              else
                  return ctx->registerTimer(activityId, name);
          }
      private:
          CRoxieServerActivity &activity;
      } interceptedCtx;
  
      const IRoxieServerActivityFactory *factory;
      IRoxieServerActivityCopyArray dependencies;
      IntArray dependencyIndexes;
      IntArray dependencyControlIds;
      IArrayOf<IActivityGraph> childGraphs;
      CachedOutputMetaData meta;
      IHThorArg *colocalParent;
      IEngineRowAllocator *rowAllocator;
      CriticalSection statecrit;
      CriticalSection statscrit;
  
      mutable CRuntimeStatisticCollection stats;
      MapStringToMyClass<ThorSectionTimer> functionTimers;
      ActivityTimeAccumulator activityStats;
      IProbeManager *probeManager = NULL;
      unsigned processed;
      unsigned numStarts = 0;
      unsigned activityId;
      activityState state;
      bool createPending;
      bool debugging;
      bool timeActivities;
      bool aborted;
      bool connected = false;
      bool collectFactoryStatistics = false;
  
  public:
      IMPLEMENT_IINTERFACE_USING(CInterfaceOf<IRoxieServerActivity>)
  
      CRoxieServerActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : basehelper(_factory->getHelper()),
            ctx(_ctx),
            interceptedCtx(*this),
            factory(_factory),
            stats(_factory->queryStatsMapping()),
            probeManager(_probeManager),
            activityId(_factory->queryId())
      {
          input = NULL;
          sourceIdx = 0;
          inputStream = NULL;
          meta.set(basehelper.queryOutputMeta());
          processed = 0;
          state=STATEreset;
          rowAllocator = NULL;
          debugging = _probeManager != NULL; // Don't want to collect timing stats from debug sessions
          colocalParent = NULL;
          createPending = true;
          timeActivities = defaultTimeActivities;
          collectFactoryStatistics = defaultCollectFactoryStatistics && !debugging && factory;
          aborted = false;
      }
      
      CRoxieServerActivity(IRoxieAgentContext *_ctx, IHThorArg & _helper)
      : basehelper(_helper), ctx(_ctx),
        interceptedCtx(*this), factory(NULL), stats(allStatistics)
      {
          activityId = 0;
          input = NULL;
          sourceIdx = 0;
          inputStream = NULL;
          ctx = NULL;
          meta.set(basehelper.queryOutputMeta());
          processed = 0;
          state=STATEreset;
          rowAllocator = NULL;
          debugging = false;
          colocalParent = NULL;
          createPending = true;
          timeActivities = defaultTimeActivities;
          collectFactoryStatistics = false; // since no factory
          aborted = false;
      }
  
      ~CRoxieServerActivity()
      {
          basehelper.Release();
          ::Release(rowAllocator);
      }
  
      virtual void beforeDispose() override
      {
          if (traceStartStop)
          {
              // There was an old comment here stating // Note- CTXLOG may not be safe
              // There were problems in the destruction order of graphs that might mean an IndirectAgentContext
              // was released while activities still referenced it.
              // However these should now all be fixed
              DBGLOG("%p destroy %d state=%s", this, activityId, queryStateText(state));
              if (watchActivityId && watchActivityId==activityId)
              {
                  DBGLOG("WATCH: %p destroy %d state=%s", this, activityId, queryStateText(state));
              }
          }
          if (state!=STATEreset)
          {
              DBGLOG("STATE: Activity %d destroyed but not reset", activityId);
              state = STATEreset;  // bit pointless but there you go... 
          }
      }
  
      const IResolvedFile *resolveLFNIndex(const char *filename, bool isOpt, bool isPrivilegedUser)
      {
          const IResolvedFile *ret = resolveLFN(filename, isOpt, isPrivilegedUser);
          if (ret && !ret->isKey())
              throw MakeStringException(0, "Attempting to read flat file as an index: %s", filename);
          return ret;
      }
  
      const IResolvedFile *resolveLFNFlat(const char *filename, bool isOpt, bool isPrivilegedUser)
      {
          const IResolvedFile *ret = resolveLFN(filename, isOpt, isPrivilegedUser);
          if (ret && ret->isKey())
              throw MakeStringException(0, "Attempting to read index as a flat file: %s", filename);
          return ret;
      }
  
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override
      {
          if (!factory)
              return;
  
          //Collate the stats for this activity from various different sources.
          CRuntimeStatisticCollection mergedStats(stats.queryMapping());
          gatherStats(mergedStats);
  
          //Because subgraphs are flattened in roxie the subgraph needs to be selected for each activity
          if (statsBuilder)
              statsBuilder->beginSubGraphScope(factory->querySubgraphId());
  
          //Still needed even if statsBuilder is null to update values in the factory
          updateEdgeStats(statsBuilder);
  
          if (statsBuilder)
          {
              StatsActivityScope ac(*statsBuilder, activityId);
              mergedStats.recordStatistics(*statsBuilder, false);
  
              //close the subgraph scope from the previous if()
              statsBuilder->endScope();
          }
  
          //Update the statistics accumulated over all the queries
          if (collectFactoryStatistics)
              factory->mergeStats(mergedStats);
  
          //Updates the query summary statistics
          if (ctx)
              ctx->queryCodeContext()->queryContextLogger().mergeStats(mergedStats);
  
          ForEachItemIn(i, childGraphs)
              childGraphs.item(i).gatherStatistics(statsBuilder);
      }
  
      virtual const IRoxieContextLogger &queryLogCtx()const
      {
          return *this;
      }
      inline const StatisticsMapping & queryStatsMapping() const
      {
          return factory ? factory->queryStatsMapping() : actStatistics;
      }
      virtual bool collectingDetailedStatistics() const
      {
          return ctx && ctx->collectingDetailedStatistics();
      }
      virtual void mergeStats(MemoryBuffer &buf)
      {
          stats.deserializeMerge(buf);
      }
      virtual void mergeStats(const CRuntimeStatisticCollection & childStats)
      {
          CriticalBlock b(statscrit);
          stats.merge(childStats);
      }
      virtual ISectionTimer *registerTimer(unsigned _activityId, const char * name)
      {
          CriticalBlock b(statscrit); // reuse statscrit to protect functionTimers - it will not be held concurrently
          ISectionTimer *timer = functionTimers.getValue(name);
          if (!timer)
          {
              timer = ThorSectionTimer::createTimer(stats, name);
              functionTimers.setValue(name, timer);
              timer->Release(); // Value returned is not linked
          }
          return timer;
      }
      void mergeStrandStats(unsigned strandProcessed, const ActivityTimeAccumulator & strandCycles)
      {
          CriticalBlock cb(statscrit);
          processed += strandProcessed;
          activityStats.merge(strandCycles);
      }
      inline void ensureRowAllocator()
      {
          if (!rowAllocator) 
              rowAllocator = createRowAllocator(meta.queryOriginal());
      }
  
      virtual IEngineRowAllocator * createRowAllocator(IOutputMetaData * metadata)
      {
          return ctx->getRowAllocatorEx(metadata, activityId, factory->getHeapFlags());
      }
  
      virtual IEngineRowAllocator * createRowAllocatorEx(IOutputMetaData * metadata, roxiemem::RoxieHeapFlags extraFlags)
      {
          return ctx->getRowAllocatorEx(metadata, activityId, (roxiemem::RoxieHeapFlags)(factory->getHeapFlags()|extraFlags));
      }
  
      inline ICodeContext *queryCodeContext()
      {
          return ctx->queryCodeContext();
      }
  
      // MORE - most of this is copied from ccd.hpp - can't we refactor?
      virtual void CTXLOGa(TracingCategory category, const char *prefix, const char *text) const
      {
          if (ctx)
              ctx->CTXLOGa(category, prefix, text);
          else
              DBGLOG("[%s] %s", prefix, text);
      }
  
      virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute__((format(printf,6,0)))
      {
          if (ctx)
              ctx->CTXLOGaeva(E, file, line, prefix, format, args);
          else
          {
              StringBuffer ss;
              ss.appendf("[%s] ERROR", prefix);
              if (E)
                  ss.append(": ").append(E->errorCode());
              if (file)
                  ss.appendf(": %s(%d) ", file, line);
              if (E)
                  E->errorMessage(ss.append(": "));
              if (format)
              {
                  ss.append(": ").valist_appendf(format, args);
              }
              LOG(MCoperatorProgress, unknownJob, "%s", ss.str());
          }
      }
  
      virtual void CTXLOGl(LogItem *log) const
      {
          if (ctx)
              ctx->CTXLOGl(log);
          else
          {
              assert(ctx);
              log->Release(); // Should never happen
          }
      }
      virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const
      {
          stats.addStatistic(kind, value);
      }
      virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const
      {
          stats.setStatistic(kind, value);
      }
      virtual void mergeStats(const CRuntimeStatisticCollection &from) const
      {
          stats.merge(from);
      }
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          merged.merge(stats);
          if (rowAllocator)
              rowAllocator->gatherStats(merged);
  
          activityStats.addStatistics(merged);
          merged.mergeStatistic(StCycleLocalExecuteCycles, queryLocalCycles());
      }
  
      virtual StringBuffer &getLogPrefix(StringBuffer &ret) const
      {
          if (ctx)
              ctx->getLogPrefix(ret);
          return ret.append('@').append(activityId);
      }
  
      virtual bool isIntercepted() const
      {
          return ctx ? ctx->isIntercepted() : false;
      }
  
      virtual bool isBlind() const
      {
          return ctx ? ctx->isBlind() : blindLogging;
      }
  
      virtual unsigned queryTraceLevel() const 
      {
          if (ctx)
              return ctx->queryTraceLevel();
          else
              return traceLevel;
      }
  
      virtual void setGlobalId(const char *id, SocketEndpoint&ep, unsigned pid) override
      {
          if (ctx)
              ctx->setGlobalId(id, ep, pid);
      }
      virtual void setCallerId(const char *id) override
      {
          if (ctx)
              ctx->setCallerId(id);
      }
      virtual const char *queryGlobalId() const
      {
          return ctx ? ctx->queryGlobalId() : nullptr;
      }
      virtual const char *queryCallerId() const override
      {
          return ctx ? ctx->queryCallerId() : nullptr;
      }
      virtual const char *queryLocalId() const
      {
          return ctx ? ctx->queryLocalId() : nullptr;
      }
      virtual void setHttpIdHeaders(const char *global, const char *caller)
      {
          if (ctx)
              ctx->setHttpIdHeaders(global, caller);
      }
      virtual const char *queryGlobalIdHttpHeader() const
      {
          return ctx ? ctx->queryGlobalIdHttpHeader() : "HPCC-Global-Id";
      }
      virtual const char *queryCallerIdHttpHeader() const
      {
          return ctx ? ctx->queryCallerIdHttpHeader() : "HPCC-Caller-Id";
      }
  
      virtual bool isPassThrough()
      {
          return false;
      }
  
      virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser)
      {
          return ctx->resolveLFN(filename, isOpt, isPrivilegedUser);
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          throwUnexpected(); // should be implemented in more derived class by anyone that has a remote adaptor
          return NULL;
      }
  
      virtual void serializeSkipInfo(MemoryBuffer &out, unsigned seekLen, const void *rawSeek, unsigned numFields, const void * seek, const SmartStepExtra &stepExtra) const
      {
          throwUnexpected(); // should be implemented in more derived class wherever needed
      }
  
      virtual IRoxieAgentContext *queryContext()
      {
          return ctx;
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return junction; }
      virtual IRoxieServerActivity *queryActivity() { return this; }
      virtual IIndexReadActivityInfo *queryIndexReadActivity() { return NULL; }
  
      virtual bool needsAllocator() const { return false; }
  
      void _onCreate(IHThorArg *_colocalParent, unsigned _numParallel)
      {
          colocalParent = _colocalParent;
          createPending = true;
          if (needsAllocator())
              ensureRowAllocator();
          processed = 0;
          if (factory)
              factory->onCreateChildQueries(ctx, &basehelper, childGraphs, this, probeManager, *this, _numParallel);
          if (ctx)
          {
              timeActivities = ctx->queryOptions().timeActivities;
              collectFactoryStatistics = ctx->queryOptions().collectFactoryStatistics && !debugging && factory;
          }
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          _onCreate(_colocalParent, 1);
      }
  
      virtual void serializeCreateStartContext(MemoryBuffer &out)
      {
          //This should only be called after onStart has been called on the helper
          assertex(!createPending);
          assertex(state==STATEstarted);
          unsigned startlen = out.length();
          basehelper.serializeCreateContext(out);
          basehelper.serializeStartContext(out);
          if (queryTraceLevel() > 10)
              CTXLOG("serializeCreateStartContext for %d added %d bytes", activityId, out.length()-startlen);
      }
  
      virtual void serializeExtra(MemoryBuffer &out) {}
  
      inline void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CriticalBlock cb(statecrit);
          if (state != STATEreset && state != STATEstarting)
          {
              CTXLOG("STATE: Expected state to be reset, but was %s, in activity %d", queryStateText(state), activityId);
          }
          state=STATEstarted;
          numStarts++;
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
          {
              CTXLOG("start %p %d", this, activityId);
              if (watchActivityId && watchActivityId==activityId)
              {
                  CTXLOG("WATCH: start %p %d", this, activityId);
              }
          }
  #endif
          executeDependencies(parentExtractSize, parentExtract, 0);
          if (input)
              input->start(parentExtractSize, parentExtract, paused);
          ensureCreated();
          basehelper.onStart(parentExtract, NULL);
          if (collectFactoryStatistics)
              factory->noteStarted();
          startJunction(junction);
      }
  
      void executeDependencies(unsigned parentExtractSize, const byte *parentExtract, unsigned controlId)
      {
          //MORE: Create a filtered list and then use asyncfor
          ForEachItemIn(idx, dependencies)
          {
              if (dependencyControlIds.item(idx) == (int) controlId)
                  dependencies.item(idx).execute(parentExtractSize, parentExtract);
          }
      }
  
      void stopDependencies(unsigned parentExtractSize, const byte *parentExtract, unsigned controlId)
      {
          ForEachItemIn(idx, dependencies)
          {
              if (dependencyControlIds.item(idx) == (int) controlId)
                  dependencies.item(idx).stop();
          }
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return activityStats.totalCycles;
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 ret = activityStats.totalCycles;
          if (input) ret -= input->queryTotalCycles();
          if (ret < 0) 
              ret = 0;
          return ret;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0) 
              return input;
          else
              return NULL;
      }
  
      inline void ensureCreated()
      {
          if (createPending)
          {
              createPending = false;
              interceptedCtx.set(ctx->queryCodeContext());
              basehelper.onCreate(&interceptedCtx, colocalParent, NULL);
          }
      }
  
      virtual void stop()
      {
          // NOTE - don't be tempted to skip the stop for activities that are reset - splitters need to see the stops
          if (state != STATEstopped)
          {
              CriticalBlock cb(statecrit);
              if (state != STATEstopped)
              {
  #ifdef TRACE_STARTSTOP
                  if (traceStartStop)
                  {
                      CTXLOG("stop %p %d (state currently %s)", this, activityId, queryStateText(state));
                      if (watchActivityId && watchActivityId==activityId)
                      {
                          CTXLOG("WATCH: stop %p %d", this, activityId);
                      }
                  }
  #endif
                  state=STATEstopped;
                  // NOTE - this is needed to ensure that dependencies which were not used are properly stopped
                  ForEachItemIn(idx, dependencies)
                  {
                      if (dependencyControlIds.item(idx) == 0)
                          dependencies.item(idx).stopSink(dependencyIndexes.item(idx));
                  }
                  if (inputStream)
                      inputStream->stop();
              }
          }
      }
  
      virtual void abort()
      {
          aborted = true;
          // MORE - should abort dependencies here?
          try
          {
              stop();
          }
          catch (IException *E)
          {
              EXCLOG(E);
              ::Release(E);
          }
      }
  
      inline void reset()
      {
          if (state != STATEreset)
          {
              CriticalBlock cb(statecrit);
              if (state != STATEreset)
              {
                  if (state==STATEstarted || state==STATEstarting)
                  {
                      if (ctx && ctx->queryServerContext() && ctx->queryCodeContext())
                      {
                          if (ctx->queryServerContext()->okToLogStartStopError()) // && traceLevel ?
                          {
                              VStringBuffer err("STATE: activity %d reset without stop", activityId);
                              ctx->queryCodeContext()->addWuException(err.str(), ROXIE_INTERNAL_ERROR, SeverityError, "roxie");
                          }
                      }
                      if (ctx && ctx->queryOptions().failOnLeaks)
                          throw makeStringExceptionV(ROXIE_INTERNAL_ERROR, "STATE: activity %d reset without stop", activityId);
                      try
                      {
                          stop();
                      }
                      catch (IException *E)
                      {
                          EXCLOG(E, "Unexpected exception in stop() called from reset()");
                          printStackReport();
                          ::Release(E);
                      }
                      catch (...)
                      {
                          DBGLOG("Unexpected unknown exception in stop() called from reset()");
                          printStackReport();
                      }
                  }
                  state = STATEreset;
  #ifdef TRACE_STARTSTOP
                  if (traceStartStop)
                  {
                      CTXLOG("reset %p %d", this, activityId);
                      if (watchActivityId && watchActivityId==activityId)
                      {
                          CTXLOG("WATCH: reset %p %d", this, activityId);
                      }
                  }
  #endif
                  resetJunction(junction);
                  ForEachItemIn(idx, dependencies)
                      dependencies.item(idx).reset();
                  if (input)
                      input->reset();
              }
          }
          aborted = false;
      }
  
      virtual void addDependency(IRoxieServerActivity &source, unsigned sourceIdx, int controlId) 
      {
          dependencies.append(source);
          dependencyIndexes.append(sourceIdx);
          dependencyControlIds.append(controlId);
      } 
  
      virtual void resetEOF()
      {
          //would make more sense if the default implementation (and eof member) were in the base class
      }
  
      // Sink activities should override this....
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract) 
      {
          throw MakeStringException(ROXIE_SINK, "Internal error: execute() requires a sink");
      }
  
      virtual void executeChild(size32_t & retSize, void * & ret, unsigned parentExtractSize, const byte * parentExtract)
      {
          throw MakeStringException(ROXIE_SINK, "Internal error: executeChild() requires a suitable sink");
      }
  
      virtual void stopSink(unsigned idx)
      {
          throw MakeStringException(ROXIE_SINK, "Internal error: stopSink() requires a suitable sink");
      }
      virtual void connectInputStreams(bool consumerOrdered)
      {
          if (input && !inputStream)
              inputStream = connectSingleStream(ctx, input, sourceIdx, junction, isInputOrdered(consumerOrdered, 0));
          if (!connected)
          {
              connectDependencies();
              connected = true;
          }
      }
  
      void connectDependencies()
      {
          ForEachItemIn(i, dependencies)
              dependencies.item(i).connectInputStreams(true);
      }
  
      virtual __int64 evaluate() 
      {
          throw MakeStringException(ROXIE_SINK, "Internal error: evaluate() requires a function");
      }
  
      virtual IFinalRoxieInput * querySelectOutput(unsigned id)
      {
          return NULL;
      }
  
      virtual bool querySetStreamInput(unsigned id, unsigned _sourceIdx, IFinalRoxieInput * _input)
      {
          return false;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          assertex(!idx);
          input = _in;
          sourceIdx = _sourceIdx;
          inputStream = NULL;
      }
  
      bool isInputOrdered(bool consumerOrdered, unsigned idx) const
      {
          if (!factory)
              return true;
          //Grouping implies the stream processing must be ordered - I think this always applies
          IFinalRoxieInput * curInput = queryInput(idx);
          if (curInput && curInput->queryOutputMeta()->isGrouped())
              return true;
          return factory->isInputOrdered(consumerOrdered, idx);
      }
  
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          assertex(!idx);
          // By default, activities are assumed NOT to support streams
          bool inputOrdered = isInputOrdered(consumerOrdered, 0);
          connectInputStreams(inputOrdered);
          // Return a single stream
          streams.append(this);
          return NULL;
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx == (unsigned) -1)
              idx = 0;
          return idx ? NULL : this;
      }
  
      virtual IOutputMetaData *queryOutputMeta() const
      { 
          return meta.queryOriginal(); 
      } 
  
      virtual unsigned queryId() const
      {
          return activityId;
      }
  
      virtual unsigned querySubgraphId() const
      {
          return factory->querySubgraphId();
      }
  
      virtual void checkAbort()
      {
          ctx->checkAbort();
      }
  
      IException *makeWrappedException(IException *e)
      {
          StringBuffer msg;
          ThorActivityKind activityKind = factory ? factory->getKind() : TAKnone;
          CTXLOG("makeWrappedException - %s (in %s %d)", e->errorMessage(msg).str(), getActivityText(activityKind), activityId); 
          if (QUERYINTERFACE(e, CWrappedException) ||  QUERYINTERFACE(e, IUserException))
              return e;
          else
              return new CWrappedException(e, activityKind, activityId);
      }
  
      virtual void gatherIterationUsage(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract)
      {
      }
  
      virtual void associateIterationOutputs(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract, IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes)
      {
      }
  
      virtual void resetOutputsUsed()
      {
      }
  
      virtual void noteOutputUsed()
      {
      }
  
      virtual const IRoxieServerActivityFactory *queryFactory() const
      {
          return factory;
      }
  
      virtual void updateEdgeStats(IStatisticGatherer * statsBuilder) const
      {
          addEdgeStats(statsBuilder, 0, numStarts, processed, 1);
      }
  
      inline ThorActivityKind getKind() const
      {
          return factory->getKind();
      }   
  
      inline bool isSink() const
      {
          return (factory != NULL) && factory->isSink();
      }
  
      void addEdgeStats(IStatisticGatherer * statsBuilder, unsigned oid, unsigned starts, unsigned _processed, unsigned _strands) const
      {
          if (statsBuilder)
          {
              StatsEdgeScope scope(*statsBuilder, activityId, oid);
              if (_strands)
                  statsBuilder->addStatistic(StNumStrands, _strands);
              if (starts != 0)
              {
                  statsBuilder->addStatistic(StNumRowsProcessed, _processed);
                  statsBuilder->addStatistic(StNumStarts, starts);
                  // Assume number of starts = number of stops since stats only ever gathered when a query is complete
                  statsBuilder->addStatistic(StNumStops, starts);
                  statsBuilder->addStatistic(StNumSlaves, 1);  // Arguable
              }
          }
  
          if (collectFactoryStatistics)
              factory->noteProcessed(oid, processed);
      }
  
      virtual void noteLibrary(IQueryFactory *library) override
      {
          throwUnexpected();
      }
  
  protected:
      RecordTranslationMode getEnableFieldTranslation() const
      {
          return factory->getEnableFieldTranslation();
      }
  };
  
  //=====================================================================================================
  
  class CRoxieServerLateStartActivity : public CRoxieServerActivity
  {
  protected:
      bool prefiltered;
      bool eof;
  
      void lateStart(unsigned parentExtractSize, const byte *parentExtract, bool any)
      {
          prefiltered = !any;
          eof = prefiltered;
          if (!prefiltered)
          {
              input->start(parentExtractSize, parentExtract, false);
              startJunction(junction);
          }
          else
          {
              if (traceStartStop)
                  CTXLOG("lateStart activity stopping input early as prefiltered");
              inputStream->stop();
          }
      }
  
  public:
  
      CRoxieServerLateStartActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          prefiltered = false;
          eof = false;
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          IFinalRoxieInput *saveInput = input;
          Owned<IStrandJunction> saveJunction = junction.getClear();
          input = NULL;   // Make sure parent does not start the chain yet
          try
          {
              CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          }
          catch (...)
          {
              // Make sure we restore these even if there is an exception thrown during start
              input = saveInput;
              junction.setown(saveJunction.getClear());
              throw;
          }
          input = saveInput;
          junction.setown(saveJunction.getClear());
      }
      virtual void stop()
      {
          if (!prefiltered && inputStream)
          {
              inputStream->stop();
          }
          else if (traceStartStop)
              CTXLOG("lateStart activity NOT stopping input late as prefiltered");
          IEngineRowStream *save = inputStream;
          inputStream = NULL;     // We already stopped inputStream - prevent parent from doing so again
          CRoxieServerActivity::stop();
          inputStream = save;
      }
  
      virtual void reset()
      {
          CRoxieServerActivity::reset();
          prefiltered = false;
      }
  };
  
  //=================================================================================
  
  class StrandOptions
  {
      // Typically set from hints, common to many stranded activities
  public:
      explicit StrandOptions(IPropertyTree &_graphNode)
      {
          //PARALLEL(1) can be used to explicitly disable parallel processing.
          numStrands = _graphNode.getPropInt("att[@name='parallel']/@value", 0);
          if ((numStrands == minus1U) || (numStrands > MAX_SENSIBLE_STRANDS))
              numStrands = getAffinityCpus();
          blockSize = _graphNode.getPropInt("hint[@name='strandblocksize']/@value", 0);
      }
      StrandOptions(const StrandOptions &from, IRoxieAgentContext *ctx)
      {
          numStrands = from.numStrands;
          blockSize = from.blockSize;
  
          if (!blockSize)
              blockSize = ctx->queryOptions().strandBlockSize;
          if (numStrands == 0)
              numStrands = ctx->queryOptions().forceNumStrands;
      }
  public:
      unsigned numStrands = 0; // if 1 it forces single-stranded operations.  (Useful for testing.)
      unsigned blockSize = 0;
  };
  
  class StrandProcessor : public CInterfaceOf<IEngineRowStream>
  {
  protected:
      CRoxieServerActivity &parent;
      IEngineRowAllocator *rowAllocator;
      IEngineRowStream *inputStream;
      ActivityTimeAccumulator activityStats;
      mutable CRuntimeStatisticCollection stats;
      unsigned processed = 0;
      unsigned numProcessedLastGroup = 0;
      const bool timeActivities;
      bool stopped = false;
      std::atomic<bool> abortRequested;
  
  public:
      explicit StrandProcessor(CRoxieServerActivity &_parent, IEngineRowStream *_inputStream, bool needsAllocator)
        : parent(_parent), inputStream(_inputStream), stats(parent.queryStatsMapping()), timeActivities(_parent.timeActivities), abortRequested(false)
      {
          if (needsAllocator)
          {
              rowAllocator = parent.createRowAllocatorEx(parent.queryOutputMeta(), roxiemem::RHFunique);
          }
          else
              rowAllocator = NULL;
      }
      ~StrandProcessor()
      {
          ::Release(rowAllocator);
      }
      virtual void start()
      {
          processed = 0;
          numProcessedLastGroup = 0;
          activityStats.reset();
      }
      virtual void stop()
      {
          if (!stopped)
          {
              if (inputStream)
                  inputStream->stop();
  
              parent.stop();
              //It would be preferrable to move activityStats (+processed?) to gatherStats(), but because of
              //the way firstRow is associated with startCycles it would require an extra parameter
              //which is not consistent with the other gatherStats() calls, or extra logic in the stats classes.
              parent.mergeStrandStats(processed, activityStats);
          }
          stopped = true;
      }
      virtual void reset()
      {
          stopped = false;
          abortRequested.store(false, std::memory_order_relaxed);
      }
      virtual void resetEOF()
      {
          inputStream->resetEOF();
      }
      inline void requestAbort() { abortRequested.store(true, std::memory_order_relaxed); }
      inline bool isAborting() { return abortRequested.load(std::memory_order_relaxed); }
  
      void gatherStats(CRuntimeStatisticCollection & mergedStats) const
      {
          mergedStats.merge(stats);
          if (rowAllocator)
              rowAllocator->gatherStats(mergedStats);
      }
      const ActivityTimeAccumulator&  queryTimings() const { return activityStats; }
  };
  
  class CRoxieServerStrandedActivity : public CRoxieServerActivity
  {
  protected:
      StrandOptions strandOptions;
      IArrayOf<StrandProcessor> strands;
      Owned<IStrandBranch> branch;
      Owned<IStrandJunction> splitter;
      Owned<IStrandJunction> sourceJunction; // A junction applied to the output of a source activity
      std::atomic<unsigned> active;
  public:
      CRoxieServerStrandedActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const StrandOptions &_strandOptions)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            strandOptions(_strandOptions, ctx)
      {
          active = 0;
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          ForEachItemIn(i, strands)
              strands.item(i).gatherStats(merged);
      }
  
      virtual void onCreate(IHThorArg *_colocalArg)
      {
          CRoxieServerActivity::_onCreate(_colocalArg, strands.ordinality());
      }
  
      //This function is pure (But also implemented out of line) to force the derived classes to implement it.
      //After calling the base class start method, and initialising any values from the helper they must call onStartStrands(),
      //this must also happen before any rows are read from the strands (e.g., by a source junction)
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) = 0;
  
      virtual void reset()
      {
          assertex(active==0);
          activityStats.reset();
          ForEachItemIn(idx, strands)
          {
              strands.item(idx).reset();
              activityStats.merge(strands.item(idx).queryTimings());
          }
          resetJunction(splitter);
          CRoxieServerActivity::reset();
          resetJunction(sourceJunction);
      }
  
      virtual void stop()
      {
          // Called from the strands... which should ensure that stop is not called more than once per strand
          //The first strand to call
          if (active)
              --active;
          if (!active)
              CRoxieServerActivity::stop();
      }
  
      void requestAbort()
      {
          ForEachItemIn(idx, strands)
              strands.item(idx).requestAbort();
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          assertex(idx == 0);
          assertex(strands.empty());
          CRoxieServerActivity::connectDependencies();
  
          bool inputOrdered = isInputOrdered(consumerOrdered, idx);
          //Note, numStrands == 1 is an explicit request to disable threading
          if (consumerOptions && (consumerOptions->numStrands != 1) && (strandOptions.numStrands != 1))
          {
              //Check to see if the consumer's settings should override
              if (strandOptions.numStrands == 0)
              {
                  strandOptions.numStrands = consumerOptions->numStrands;
                  strandOptions.blockSize = consumerOptions->blockSize;
              }
              else if (consumerOptions->numStrands > strandOptions.numStrands)
              {
                  strandOptions.numStrands = consumerOptions->numStrands;
              }
          }
  
          Owned <IStrandJunction> recombiner;
          if (input)
          {
              if (strandOptions.numStrands == 1)
              {
                  // 1 means explicitly requested single-strand.
                  IEngineRowStream *instream = connectSingleStream(ctx, input, sourceIdx, junction, inputOrdered);
                  strands.append(*createStrandProcessor(instream));
              }
              else
              {
                  PointerArrayOf<IEngineRowStream> instreams;
                  recombiner.setown(input->getOutputStreams(ctx, sourceIdx, instreams, &strandOptions, inputOrdered, orderedCallbacks));
                  if ((instreams.length() == 1) && (strandOptions.numStrands != 0))  // 0 means did not specify - we should use the strands that our upstream provides
                  {
                      assertex(recombiner == NULL);
                      // Create a splitter to split the input into n... and a recombiner if need to preserve sorting
                      if (inputOrdered)
                      {
                          branch.setown(createStrandBranch(ctx->queryRowManager(), strandOptions.numStrands, strandOptions.blockSize, true, input->queryOutputMeta()->isGrouped(), false, orderedCallbacks));
                          splitter.set(branch->queryInputJunction());
                          recombiner.set(branch->queryOutputJunction());
                      }
                      else
                      {
                          splitter.setown(createStrandJunction(ctx->queryRowManager(), 1, strandOptions.numStrands, strandOptions.blockSize, false));
                      }
                      splitter->setInput(0, instreams.item(0));
                      for (unsigned strandNo = 0; strandNo < strandOptions.numStrands; strandNo++)
                          strands.append(*createStrandProcessor(splitter->queryOutput(strandNo)));
                  }
                  else
                  {
                      // Ignore my hint and just use the width already split into...
                      ForEachItemIn(strandNo, instreams)
                          strands.append(*createStrandProcessor(instreams.item(strandNo)));
                  }
              }
          }
          else
          {
              unsigned numStrands = strandOptions.numStrands ? strandOptions.numStrands : 1;
              for (unsigned i=0; i < numStrands; i++)
                  strands.append(*createStrandSourceProcessor(inputOrdered));
  
              if (inputOrdered && (numStrands > 1))
              {
                  if (consumerOptions)
                  {
                      //If the output activities are also stranded then need to create a version of the branch
                      bool isGrouped = queryOutputMeta()->isGrouped();
                      branch.setown(createStrandBranch(ctx->queryRowManager(), strandOptions.numStrands, strandOptions.blockSize, true, isGrouped, true, orderedCallbacks));
                      sourceJunction.set(branch->queryInputJunction());
                      recombiner.set(branch->queryOutputJunction());
                      assertex((orderedCallbacks && !recombiner) || (!orderedCallbacks && recombiner));
  
                      //This is different from the branch above.  The first "junction" has the source activity as the input, and the outputs as the result of the activity
                      for (unsigned strandNo = 0; strandNo < strandOptions.numStrands; strandNo++)
                      {
                          sourceJunction->setInput(strandNo, &strands.item(strandNo));
                          streams.append(sourceJunction->queryOutput(strandNo));
                      }
  #ifdef TRACE_STRANDS
                      if (traceLevel > 2)
                          DBGLOG("Executing activity %u with %u strands", activityId, strands.ordinality());
  #endif
                      return recombiner.getClear();
                  }
                  else
                  {
                      //Feeding into a non threaded activity, so create a M:1 junction to combine the source strands
                      recombiner.setown(createStrandJunction(ctx->queryRowManager(), numStrands, 1, strandOptions.blockSize, inputOrdered));
                  }
              }
          }
          ForEachItemIn(i, strands)
              streams.append(&strands.item(i));
  #ifdef TRACE_STRANDS
          if (traceLevel > 2)
              DBGLOG("Executing activity %u with %u strands", activityId, strands.ordinality());
  #endif
  
          return recombiner.getClear();
      }
  
      virtual StrandProcessor *createStrandProcessor(IEngineRowStream *instream) = 0;
  
      //MORE: Possibly this class should be split into two for sinks and non sinks...
      virtual StrandProcessor *createStrandSourceProcessor(bool inputOrdered) = 0;
  
      virtual const void * nextRow()
      {
          throwUnexpected();
      }
  
      inline unsigned numStrands() const { return strands.ordinality(); }
  
      virtual void updateEdgeStats(IStatisticGatherer * statsBuilder) const
      {
          addEdgeStats(statsBuilder, 0, numStarts, processed, strands.ordinality());
      }
  
  protected:
  
      void onStartStrands()
      {
          ForEachItemIn(idx, strands)
          {
              strands.item(idx).start();
              active++;
          }
      }
  };
  
  
  //For some reason gcc doesn't let you specify a function as pure virtual and define it at the same time.
  void CRoxieServerStrandedActivity::start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
  {
      CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      startJunction(splitter);
  }
  
  
  class CRoxieServerStrandedLateStartActivity : public CRoxieServerStrandedActivity
  {
  protected:
      bool prefiltered;
      bool eof;
  
      void lateStart(unsigned parentExtractSize, const byte *parentExtract, bool any)
      {
          prefiltered = !any;
          eof = prefiltered;
          if (!prefiltered)
          {
              //At this point
              input->start(parentExtractSize, parentExtract, false);
              startJunction(splitter);
              onStartStrands();  // Initialise the strands - the helper will have been correctly initialized by this point.
              startJunction(sourceJunction);
          }
          else
          {
              if (traceStartStop)
                  CTXLOG("strandedLateStart activity stopping input early as prefiltered");
              ForEachItemIn(idx, strands)
                  strands.item(idx).stop();
          }
      }
  
  public:
  
      CRoxieServerStrandedLateStartActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const StrandOptions &_strandOptions)
          : CRoxieServerStrandedActivity(_ctx, _factory, _probeManager, _strandOptions)
      {
          prefiltered = false;
          eof = false;
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          IFinalRoxieInput *save = input;
          input = NULL;   // Make sure parent does not start the chain yet - but we do want to do the dependencies (because the decision about whether to start may depend on them)
          try
          {
              CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          }
          catch (...)
          {
              // Make sure we restore these even if there is an exception thrown during start
              input = save;
              throw;
          }
          input = save;
      }
  
      virtual void reset()
      {
          CRoxieServerStrandedActivity::reset();
          prefiltered = false;
      }
  
  };
  
  //=====================================================================================================
  
  static std::atomic<unsigned> nextInstanceId;
  
  extern unsigned getNextInstanceId()
  {
      return ++nextInstanceId;
  }
  
  static std::atomic<unsigned> nextRuid;
  
  ruid_t getNextRuid()
  {
      ruid_t ret = ++nextRuid;
      while (ret < RUID_FIRST)
          ret = ++nextRuid; // ruids 0 and 1 are reserved for pings/unwanted discarder.
      return ret;
  }
  
  void setStartRuid(unsigned restarts)
  {
      nextRuid = restarts * 0x10000;
      nextInstanceId = restarts * 10000;
  }
  
  enum { LimitSkipErrorCode = 0, KeyedLimitSkipErrorCode = 1 };
  
  class DECL_EXCEPTION LimitSkipException : public IException, public CInterface
  {
      int code;
  public:
      LimitSkipException(int _code) { code = _code; }
      IMPLEMENT_IINTERFACE;
      virtual int             errorCode() const { return code; }
      virtual StringBuffer &  errorMessage(StringBuffer &msg) const { return msg.append("LimitSkipException"); }
      virtual MessageAudience errorAudience() const { return MSGAUD_programmer; }
  };
  
  IException *makeLimitSkipException(bool isKeyed)
  {
      // We need to make sure what we throw is IException not something derived from it....
      return new LimitSkipException(isKeyed ? KeyedLimitSkipErrorCode : LimitSkipErrorCode);
  }
  
  //=================================================================================
  
  interface IRecordPullerCallback : extends IExceptionHandler
  {
      virtual void processRow(const void *row) = 0;
      virtual void processEOG() = 0;
      virtual void processGroup(const ConstPointerArray &rows) = 0;
      virtual void processDone() = 0;
  };
  
  class RecordPullerThread : public RestartableThread
  {
  protected:
      IFinalRoxieInput *input;
      Owned<IStrandJunction> junction;
      unsigned sourceIdx = 0;
      IEngineRowStream *inputStream;
      IRecordPullerCallback *helper;
      Semaphore started;                      // MORE: GH->RKC I'm pretty sure this can be deleted, since handled by RestartableThread
      bool groupAtOnce, eog;
      std::atomic<bool> eof;
      CriticalSection crit;
  
  public:
      RecordPullerThread(bool _groupAtOnce) 
          : RestartableThread("RecordPullerThread"), groupAtOnce(_groupAtOnce)
      {
          input = NULL;
          inputStream = NULL;
          helper = NULL;
          eof = eog = FALSE;
      }
  
      inline unsigned __int64 queryTotalCycles() const
      {
          return input->queryTotalCycles();
      }
  
      void setInput(IRecordPullerCallback *_helper, unsigned _sourceIdx, IFinalRoxieInput *_input)
      {
          helper = _helper;
          input = _input;
          sourceIdx = _sourceIdx;
      }
  
      IFinalRoxieInput *queryInput() const
      {
          return input;
      }
  
      virtual void connectInputStreams(IRoxieAgentContext *ctx, bool consumerOrdered)
      {
          inputStream = connectSingleStream(ctx, input, sourceIdx, junction, consumerOrdered);
      }
  
      IEngineRowStream *queryStream() const
      {
          return inputStream;
      }
  
      void start(unsigned parentExtractSize, const byte *parentExtract, bool paused, unsigned preload, bool noThread, IRoxieAgentContext *ctx)
      {
          eof = false;
          eog = false;
          input->start(parentExtractSize, parentExtract, paused);
          startJunction(junction);
          try
          {
              if (preload && !paused)
              {
                  if (traceLevel > 4)
                      DBGLOG("Preload fetching first %d records", preload);
                  if (groupAtOnce)
                      pullGroups(preload);
                  else
                      pullRecords(preload);
              }
              if (eof)
              {
                  if (traceLevel > 4)
                      DBGLOG("No need to start puller after preload");
                  helper->processDone();
              }
              else
              {
                  if (!noThread)
                  {
                      StringBuffer logPrefix("[");
                      if (ctx) ctx->getLogPrefix(logPrefix);
                      logPrefix.append("] ");
                      RestartableThread::start(logPrefix);
                      started.wait();
                  }
              }
          }
          catch (IException *e)
          {
              helper->fireException(e);
          }
          catch (...)
          {
              helper->fireException(MakeStringException(ROXIE_INTERNAL_ERROR, "Unexpected exception caught in RecordPullerThread::start"));
          }
      }
  
      void stop()
      {
          if (traceStartStop)
              DBGLOG("RecordPullerThread::stop");
          {
              CriticalBlock c(crit); // stop is called on our consumer's thread. We need to take care calling stop for our input to make sure it is not in mid-nextRow etc etc.
              if (inputStream)
                  inputStream->stop();
              eof = true;
          }
          RestartableThread::join();
      }
  
      void reset()
      {
          input->reset();
          resetJunction(junction);
      }
  
      virtual int run()
      {
          started.signal();
          try
          {
              if (groupAtOnce)
                  pullGroups((unsigned) -1);
              else
                  pullRecords((unsigned) -1);
              helper->processDone();
          }
          catch (IException *e)
          {
              helper->fireException(e);
          }
          catch (...)
          {
              helper->fireException(MakeStringException(ROXIE_INTERNAL_ERROR, "Unexpected exception caught in RecordPullerThread::run"));
          }
          return 0;
      }
  
      void done()
      {
          helper->processDone();
      }
  
      bool pullRecords(unsigned preload)
      {
          if (eof)
              return false;
          while (preload && !eof)
          {
              const void * row = nullptr;
              {
                  CriticalBlock c(crit); // See comments in stop for why this is needed
                  if (!eof)
                      row = inputStream->nextRow();
              }
              if (row)
              {
                  eog = false;
                  helper->processRow(row);
              }
              else if (!eog)
              {
                  helper->processEOG();
                  eog = true;
              }
              else
              {
                  eof = true;
                  return false;
              }
              if (preload != (unsigned) -1)
                  preload--;
          }
          return true;
      }
  
      void pullGroups(unsigned preload)
      {
          ConstPointerArray thisGroup;
          unsigned rowsDone = 0;
          while (preload && !eof)
          {
              const void *row = nullptr;
              {
                  CriticalBlock c(crit);
                  if (!eof)
                      row = inputStream->nextRow();
              }
              if (row)
              {
                  thisGroup.append(row);
                  rowsDone++;
              }
              else if (thisGroup.length())
              {
                  helper->processGroup(thisGroup);
                  thisGroup.kill();
                  if (preload != (unsigned) -1)
                  {
                      if (preload > rowsDone)
                          preload -= rowsDone;
                      else
                          break;
                  }
                  rowsDone = 0;
              }
              else
              {
                  eof = true;
                  break;
              }
          }
      }
  };
  
  
  //=================================================================================
  
  #define READAHEAD_SIZE 1000
  
  // MORE - this code copied from ThreadedConcat code - may be able to common up some.
  
  class CRoxieServerReadAheadInput : implements IEngineRowStream, implements IFinalRoxieInput, implements IRecordPullerCallback, public CInterface
  {
      QueueOf<const void, true> buffer;
      InterruptableSemaphore ready;
      InterruptableSemaphore space;
      CriticalSection crit;
      bool eof;
      bool disabled;
      IRoxieAgentContext *ctx;
      RecordPullerThread puller;
      unsigned preload;
      cycle_t totalCycles;
      bool timeActivities;
  
  public:
      IMPLEMENT_IINTERFACE;
      CRoxieServerReadAheadInput(IRoxieAgentContext *_ctx, unsigned _preload) : ctx(_ctx), puller(true), preload(_preload)
      {
          eof = false;
          disabled = (ctx->queryDebugContext() != NULL);
          totalCycles = 0;
          timeActivities = ctx->queryOptions().timeActivities;
      }
  
      virtual IRoxieServerActivity *queryActivity()
      {
          return puller.queryInput()->queryActivity();
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          assertex(!idx);
          puller.connectInputStreams(ctx, consumerOrdered);
          streams.append(this);
          return NULL;
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { return nullptr; }
  
      virtual IIndexReadActivityInfo *queryIndexReadActivity() 
      {
          return puller.queryInput()->queryIndexReadActivity();
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          if (disabled)
              puller.queryInput()->start(parentExtractSize, parentExtract, paused);
          else
          {
              space.reinit(READAHEAD_SIZE);
              ready.reinit();
              puller.start(parentExtractSize, parentExtract, paused, preload, false, ctx);
          }
      }
  
      virtual void stop()
      {
          if (disabled)
              puller.queryStream()->stop();
          else
          {
              space.interrupt();
              ready.interrupt();
              puller.stop();
          }
      }
  
      virtual void reset()
      {
          if (disabled)
              puller.queryInput()->reset();
          else
          {
              puller.reset();
              ForEachItemIn(idx1, buffer)
                  ReleaseRoxieRow(buffer.item(idx1));
              buffer.clear();
          }
      }
  
      virtual void resetEOF()
      {
          throwUnexpected();
      }
  
      virtual IOutputMetaData * queryOutputMeta() const 
      {
          return puller.queryInput()->queryOutputMeta(); 
      }
  
      void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          assertex(!idx);
          puller.setInput(this, _sourceIdx, _in);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return puller.queryInput();
          else
              return NULL;
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return totalCycles;
      }
  
      virtual const void * nextRow()
      {
          SimpleActivityTimer t(totalCycles, timeActivities);
          if (disabled)
              return puller.queryStream()->nextRow();
          else
          {
              for (;;)
              {
                  {
                      CriticalBlock b(crit);
                      if (eof && !buffer.ordinality())
                          return NULL;  // eof
                  }
                  ready.wait();
                  const void *ret;
                  {
                      CriticalBlock b(crit);
                      ret = buffer.dequeue();
                  }
                  space.signal();
                  return ret;
              }
          }
      }
  
      virtual bool fireException(IException *e)
      {
          // called from puller thread on failure
          ready.interrupt(LINK(e));
          space.interrupt(e);
          return true;
      }
  
      virtual void processRow(const void *row)
      {
          {
              CriticalBlock b(crit);
              buffer.enqueue(row);
          }
          ready.signal();
          space.wait();
      }
  
      virtual void processGroup(const ConstPointerArray &rows)
      {
          // NOTE - a bit bizarre in that it waits for the space AFTER using it.
          // But the space semaphore is only there to stop infinite readahead. And otherwise it would deadlock
          // if group was larger than max(space)
          {
              CriticalBlock b(crit);
              ForEachItemIn(idx, rows)
                  buffer.enqueue(rows.item(idx));
              buffer.enqueue(NULL);
          }
          for (unsigned i2 = 0; i2 <= rows.length(); i2++) // note - does 1 extra for the null
          {
              ready.signal();
              space.wait();
          }
      }
  
      virtual void processEOG()
      {
          // Used when output is not grouped - just ignore
      }
  
      virtual void processDone()
      {
          CriticalBlock b(crit);
          eof = true;
          ready.signal();
      }
  };
  
  //=================================================================================
  
  class CRoxieServerTwoInputActivity : public CRoxieServerActivity
  {
  protected:
      IFinalRoxieInput *input1;
      IEngineRowStream *inputStream1;
      Owned<IStrandJunction> junction1;
  
      unsigned sourceIdx1 = 0;
      Owned<CRoxieServerReadAheadInput> puller;
  
  public:
      CRoxieServerTwoInputActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          input1 = NULL;
          inputStream1 = NULL;
      }
  
      ~CRoxieServerTwoInputActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          input1->start(parentExtractSize, parentExtract, paused);
          startJunction(junction1);
      }
  
      virtual void stop()
      {
          inputStream1->stop();
          CRoxieServerActivity::stop();
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 ret;
          __int64 inputCycles = input->queryTotalCycles();
          __int64 input1Cycles = input1->queryTotalCycles();
          if (puller)
              ret = activityStats.totalCycles - (inputCycles > input1Cycles ? inputCycles : input1Cycles);
          else
              ret = activityStats.totalCycles - (inputCycles + input1Cycles);
          if (ret < 0) 
              ret = 0;
          return ret;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          switch (idx)
          {
          case 0:
              return input;
          case 1:
              return input1;
          default:
              return NULL;
          }
      }
  
      virtual void reset()    
      {
          CRoxieServerActivity::reset(); 
          if (input1)
              input1->reset();
          resetJunction(junction1);
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          switch(idx)
          {
          case 0:
              CRoxieServerActivity::setInput(idx, _sourceIdx, _in);
              break;
          case 1:
              input1 = _in;
              sourceIdx1 = _sourceIdx;
              break;
          default:
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
          }   
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          inputStream1 = connectSingleStream(ctx, input1, sourceIdx1, junction1, isInputOrdered(consumerOrdered, 1));
          //Assumes input1 will be stranded if any.
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  };
  
  //=================================================================================
  
  class CRoxieServerMultiInputBaseActivity : public CRoxieServerActivity
  {
  protected:
      unsigned numInputs;
      unsigned numStreams;
      IFinalRoxieInput **inputArray;
      unsigned *sourceIdxArray;
      IEngineRowStream **streamArray;
      Owned<IStrandJunction> *junctionArray;
  
  public:
      CRoxieServerMultiInputBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), numInputs(_numInputs)
      {
          numStreams = numInputs;
          inputArray = new IFinalRoxieInput*[numInputs];
          sourceIdxArray = new unsigned[numInputs];
          streamArray = new IEngineRowStream*[numStreams];
          junctionArray = new Owned<IStrandJunction>[numStreams];
          for (unsigned i1 = 0; i1 < numInputs; i1++)
          {
              inputArray[i1] = NULL;
              sourceIdxArray[i1] = 0;
          }
          for (unsigned i2 = 0; i2 < numStreams; i2++)
              streamArray[i2] = NULL;
      }
  
      ~CRoxieServerMultiInputBaseActivity()
      {
          delete [] inputArray;
          delete [] sourceIdxArray;
          delete [] streamArray;
          delete [] junctionArray;
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 localCycles = activityStats.totalCycles;
          for (unsigned i = 0; i < numInputs; i++)
              localCycles -= inputArray[i]->queryTotalCycles();
          if (localCycles < 0)
              localCycles = 0;
          return localCycles;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx < numInputs)
              return inputArray[idx];
          else
              return NULL;
      }
  
      virtual IFinalRoxieInput * queryConcreteInput(unsigned idx)
      {
          return queryInput(idx);
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
      {
          if (whichInput < numInputs)
              return streamArray[whichInput];
          else
              return NULL;
      }
  
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
      {
          if (idx < numInputs)
              return junctionArray[idx];
          else
              return NULL;
      }
  
      virtual void stop() override
      {
          for (unsigned idx = 0; idx < numStreams; idx++)
          {
              streamArray[idx]->stop();
          }
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          for (unsigned i = 0; i < numInputs; i++)
              inputArray[i]->reset();
          for (unsigned iS = 0; iS < numStreams; iS++)
          {
              resetJunction(junctionArray[iS]);
          }
          CRoxieServerActivity::reset(); 
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          assertex(idx < numInputs && idx < numStreams);
          inputArray[idx] = _in;
          sourceIdxArray[idx] = _sourceIdx;
      }
  
      virtual void connectInputStreams(bool consumerOrdered)
      {
          //There could be situations (e.g., NONEMPTY), where you might want to strand the activity.
          connectSingleInputStreams(consumerOrdered);
          CRoxieServerActivity::connectInputStreams(consumerOrdered);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, NULL, consumerOrdered, nullptr); // The input basesclass does not have an input
      }
  
  protected:
      void connectSingleInputStreams(bool consumerOrdered)
      {
          for (unsigned i = 0; i < numInputs; i++)
              streamArray[i] = connectSingleStream(ctx, inputArray[i], sourceIdxArray[i], junctionArray[i], consumerOrdered);
      }
  };
  
  //=================================================================================
  
  class CRoxieServerMultiInputActivity : public CRoxieServerMultiInputBaseActivity
  {
  public:
      CRoxieServerMultiInputActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerMultiInputBaseActivity::start(parentExtractSize, parentExtract, paused);
          for (unsigned i = 0; i < numInputs; i++)
          {
              inputArray[i]->start(parentExtractSize, parentExtract, paused);
          }
          for (unsigned iS = 0; iS < numStreams; iS++)
          {
              startJunction(junctionArray[iS]);
          }
      }
  
      virtual void stop()
      {
          for (unsigned i = 0; i < numStreams; i++)
          {
              streamArray[i]->stop();
          }
          CRoxieServerMultiInputBaseActivity::stop();
      }
  
  };
  
  //=====================================================================================================
  
  class CRoxieServerInternalSinkActivity : public CRoxieServerActivity
  {
  protected:
      unsigned numOutputs;
      bool executed;
      bool *stopped;
      CriticalSection ecrit;
      Owned<IException> exception;
  
  public:
      CRoxieServerInternalSinkActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numOutputs)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), numOutputs(_numOutputs)
      {
          executed = false;
          stopped = new bool[numOutputs];
          for (unsigned s = 0; s < numOutputs; s++)
              stopped[s] = false;
      }
  
      ~CRoxieServerInternalSinkActivity()
      {
          delete [] stopped;
      }
  
      virtual void reset()
      {
          for (unsigned s = 0; s < numOutputs; s++)
              stopped[s] = false;
          executed = false;
          exception.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          return NULL;
      }
  
      virtual void stopSink(unsigned outputIdx)
      {
          if (outputIdx < numOutputs && !stopped[outputIdx])  // Implicit dependencies on DiskWrite activities do not count as outputs
          {
              stopped[outputIdx] = true;
              for (unsigned s = 0; s < numOutputs; s++)
                  if (!stopped[s])
                      return;
              stop(); // all outputs stopped - stop parent.
          }
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  
      virtual void onExecute() = 0;
  
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract) 
      {
          CriticalBlock b(ecrit);
          if (exception)
              throw exception.getLink();
          if (!executed)
          {
              try
              {
                  start(parentExtractSize, parentExtract, false);
                  {
                      ActivityTimer t(activityStats, timeActivities); // unfortunately this is not really best place for seeing in debugger.
                      onExecute();
                  }
                  stop();
                  executed = true;
              }
              catch (IException *E)
              {
                  exception.set(E); // (or maybe makeWrappedException?)
                  abort();
                  throw;
              }
              catch (...)
              {
                  exception.set(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught at %s:%d", sanitizeSourceFile(__FILE__), __LINE__));
                  abort();
                  throw;
              }
          }
      }
  
  };
  
  
  //=================================================================================
  
  class CRoxieServerQueryPacket : implements IRoxieServerQueryPacket, public CInterface
  {
  protected:
      Owned<IMessageResult> result;
      Owned<IRoxieQueryPacket> packet;
      Linked<IRoxieServerQueryPacket> continuation;
      unsigned hash;
      unsigned seq;
      unsigned lastDebugSequence;
      Owned<IRoxieQueryPacket> lastDebugResponse;
  
      ILRUChain *prev;
      ILRUChain *next;
      bool delayed;
  public:
      IMPLEMENT_IINTERFACE;
      CRoxieServerQueryPacket(IRoxieQueryPacket *p) : packet(p)
      {
          hash = 0;
          seq = 0;
          prev = NULL;
          next = NULL;
          delayed = false;
          lastDebugSequence = 0;
      }
      virtual IRoxieQueryPacket *queryPacket() const
      {
          return packet;
      }
      virtual bool isContinuation() const
      {
          return packet && (packet->queryHeader().continueSequence & ~CONTINUE_SEQUENCE_SKIPTO) != 0;
      }
      virtual bool isDelayed() const
      {
          return delayed;
      }
      virtual bool isEnd() const
      {
          return false;
      }
      virtual bool isLimit(unsigned __int64 &_rowLimit, unsigned __int64 &_keyedLimit, unsigned __int64 &_stopAfter) const
      {
          return false;
      }
      virtual bool hasResult() const 
      {
          return result != NULL;
      }
      virtual void setDelayed(bool _delayed)
      {
          delayed = _delayed;
      }
      virtual void setPacket(IRoxieQueryPacket *_packet)
      {
          packet.setown(_packet);
      }
      virtual void setSequence(unsigned _seq)
      {
          assertex(!IsShared());
          seq = _seq;
      }
      virtual unsigned getSequence() const
      {
          return seq;
      }
      IMessageResult *getResult()
      {
          return result.getLink();
      }
      IMessageResult *queryResult()
      {
          return result;
      }
      void setResult(IMessageResult *r)
      {
          result.setown(r);
      }
      IRoxieServerQueryPacket *queryContinuation()
      {
          return continuation;
      }
      void setContinuation(IRoxieServerQueryPacket *c)
      {
          continuation.setown(c);
      }
      virtual unsigned queryHash() const
      {
          return hash;
      }
      virtual void setHash(unsigned _hash)
      {
          hash = _hash;
      }
      virtual ILRUChain *queryPrev() const { return prev; }
      virtual ILRUChain *queryNext() const { return next; }
      virtual void setPrev(ILRUChain *p) { prev = p; }
      virtual void setNext(ILRUChain *n) { next = n; }
      virtual void unchain()
      {
          if (prev && next)
          {
              prev->setNext(next);
              next->setPrev(prev);
          }
          next = NULL;
          prev = NULL;
      }
  
      virtual IRoxieQueryPacket *getDebugResponse(unsigned sequence)
      {
          if (sequence == lastDebugSequence)
              return lastDebugResponse.getLink();
          else if (sequence > lastDebugSequence)
          {
              lastDebugResponse.clear();
              return NULL;
          }
          else
              throwUnexpected();
      }
  
      virtual void setDebugResponse(unsigned sequence, IRoxieQueryPacket *response)
      {
          lastDebugSequence = sequence;
          lastDebugResponse.set(response);
      }
  
  };
  
  class CRoxieServerQueryPacketEndMarker : public CRoxieServerQueryPacket
  {
  public:
      CRoxieServerQueryPacketEndMarker() : CRoxieServerQueryPacket(NULL)
      {
      }
      virtual bool isEnd() const
      {
          return true;
      }
  };
  
  class CRoxieServerQueryPacketLimitMarker : public CRoxieServerQueryPacket
  {
      unsigned __int64 rowLimit;
      unsigned __int64 keyedLimit;
      unsigned __int64 stopAfter;
  
  public:
      CRoxieServerQueryPacketLimitMarker(unsigned __int64 _rowLimit, unsigned __int64 _keyedLimit, unsigned __int64 _stopAfter) : CRoxieServerQueryPacket(NULL)
      {
          rowLimit = _rowLimit;
          keyedLimit = _keyedLimit;
          stopAfter = _stopAfter;
      }
      virtual bool isLimit(unsigned __int64 &_rowLimit, unsigned __int64 &_keyedLimit, unsigned __int64 &_stopAfter) const
      {
          _rowLimit = rowLimit;
          _keyedLimit = keyedLimit;
          _stopAfter = stopAfter;
          return true;
      }
  };
  
  class CRowArrayMessageUnpackCursor : implements IMessageUnpackCursor, public CInterface
  {
      ConstPointerArray &data;
      Linked<IMessageResult> result;
  
  public:
      IMPLEMENT_IINTERFACE;
      CRowArrayMessageUnpackCursor(ConstPointerArray &_data, IMessageResult *_result)
          : data(_data), result(_result)
      {
      }
  
      virtual bool atEOF() const
      {
          return data.length()==0;
      }
  
      virtual bool isSerialized() const
      {
          return false;
      }
  
      virtual const void * getNext(int length)
      {
          if (!data.length())
              return NULL;
          const void *ret = data.item(0);
          data.remove(0);
          return ret;
      }
  
  };
  
  // MORE - should possibly move more over to the lazy version used in indexread?
  
  class CRowArrayMessageResult : implements IMessageResult, public CInterface
  {
      ConstPointerArray data;
  
  public:
      IMPLEMENT_IINTERFACE;
      CRowArrayMessageResult()
      {
      }
  
      ~CRowArrayMessageResult()
      {
          ReleaseRoxieRows(data);
      }
  
      virtual IMessageUnpackCursor *getCursor(IRowManager *rowMgr) const
      {
          CRowArrayMessageResult  *_this = (CRowArrayMessageResult  *) this;
          return new CRowArrayMessageUnpackCursor(_this->data, _this);
      }
  
      virtual const void *getMessageHeader(unsigned &length) const
      {
          throwUnexpected(); // should never get called - I don't have a header available
      }
  
      virtual const void *getMessageMetadata(unsigned &length) const
      {
          length = 0;
          return NULL;
      }
  
      virtual void discard() const
      {
          throwUnexpected();
      }
  
      void append(const void *row)
      {
          data.append(row);
      }
  };
  
  void throwRemoteException(IMessageUnpackCursor *extra)
  {
      RecordLengthType *rowlen = (RecordLengthType *) extra->getNext(sizeof(RecordLengthType));
      if (rowlen)
      {
          char *xml = (char *) extra->getNext(*rowlen);
          ReleaseRoxieRow(rowlen);
          Owned<IPropertyTree> p = createPTreeFromXMLString(xml, ipt_fast);
          ReleaseRoxieRow(xml);
          unsigned code = p->getPropInt("Code", 0);
          const char *msg = p->queryProp("Message");
          if (!msg)
              msg = xml;
          throw MakeStringException(code, "%s", msg);
      }
      throwUnexpected();
  }
  
  class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxieInput, implements IExceptionHandler, public CInterface
  {
      friend class CRemoteResultMerger;
      class CRemoteResultMerger
      {
          class HeapEntry : public CInterface
          {
          private:
              CRemoteResultAdaptor &adaptor;
              IMessageUnpackCursor *cursor;
          public:
              const void *current;
              bool isLast;
              bool lastIsComplete;
              IRoxieServerQueryPacket *packet; 
              unsigned seq;
  
          public:
              inline const void *noteResult(IMessageUnpackCursor *_cursor, bool _lastIsComplete)
              {
                  cursor = _cursor;
                  lastIsComplete = _lastIsComplete;
                  return next();
              }
  
          public:
              HeapEntry(CRemoteResultAdaptor &_adaptor, IRoxieServerQueryPacket *_packet, unsigned _seq) : adaptor(_adaptor), packet(_packet), seq(_seq)
              {
                  cursor = NULL;
                  current = NULL;
                  isLast = false;
                  lastIsComplete = true;
              }
              ~HeapEntry()
              {
                  ::Release(packet);
                  ::Release(cursor);
                  ReleaseRoxieRow(current);
              }
  
              bool isCompleteMatch() const
              {
                  if (!isLast || lastIsComplete)
                      return true;
                  else
                      return false;
              }
  
              const void *next()
              {
                  if (cursor)
                  {
                      ReleaseClearRoxieRow(current);
                      current = adaptor.getRow(cursor);
                      isLast = cursor->atEOF();
                      if (!current)
                      {
                          cursor->Release();
                          cursor = NULL;
                      }
                  }
                  return current;
              }
              unsigned skipTo(IRangeCompare *compare, const void *seek, unsigned numFields, bool requireExactMatch)
              {
                  // MORE - This loop should possibly be a binchop... though it's not absolutely clear that is true (depends on term frequencies)
                  unsigned skipped = 0;
                  for (;;)
                  {
                      int c = compare->docompare(current, seek, numFields);
  
                      //If larger than the seek values, then we may be allowed to return an inexact match, 
                      //if equal then it is required to be an exact match,
                      if (c > 0)
                      {
                          if (!requireExactMatch || isCompleteMatch())
                              break;
                      }
                      else if ((c == 0) && isCompleteMatch())
                          break;
  
                      skipped++;
                      if (!next())
                          break;
                  }
                  return skipped;
              }
  
          };
  
          CRemoteResultAdaptor &adaptor;
          CIArrayOf<HeapEntry> heapEntries;
          UnsignedArray heap;
          IRowManager *rowManager;
          unsigned numPending;
          unsigned numFields;
          bool endSeen;
          bool remakePending;
          IRangeCompare *compare;
          bool deferredContinuation;
  
          inline int doCompare(unsigned l, unsigned r)
          {
              int ret = compare->docompare(heapEntries.item(l).current, heapEntries.item(r).current, numFields);
              if (!ret) ret = heapEntries.item(l).seq - heapEntries.item(r).seq;
              return ret;
          }
  
          void makeHeap()
          {
              /* Permute blocks to establish the heap property
                 For each element p, the children are p*2+1 and p*2+2 (provided these are in range)
                 The children of p must both be greater than or equal to p
                 The parent of a child c is given by p = (c-1)/2
              */
              unsigned i;
              unsigned n = heap.length();
              unsigned *s = heap.getArray();
              for (i=1; i<n; i++)
              {
                  unsigned r = s[i];
                  int c = i; /* child */
                  while (c > 0) 
                  {
                      int p = (c-1)/2; /* parent */
                      if ( doCompare( s[c], s[p] ) >= 0 ) 
                          break;
                      s[c] = s[p];
                      s[p] = r;
                      c = p;
                  }
              }
              remakePending = false;
          }
  
          void remakeHeap()
          {
              /* The row associated with block[0] will have changed
                 This code restores the heap property
              */
              unsigned p = 0; /* parent */
              unsigned n = heap.length();
              unsigned *s = heap.getArray();
              while (1) 
              {
                  unsigned c = p*2 + 1; /* child */
                  if ( c >= n ) 
                      break;
                  /* Select smaller child */
                  if ( c+1 < n && doCompare( s[c+1], s[c] ) < 0 ) c += 1;
                  /* If child is greater or equal than parent then we are done */
                  if ( doCompare( s[c], s[p] ) >= 0 ) 
                      break;
                  /* Swap parent and child */
                  unsigned r = s[c];
                  s[c] = s[p];
                  s[p] = r;
                  /* child becomes parent */
                  p = c;
              }
              remakePending = false;
          }
  
          void append(IRoxieServerQueryPacket *p, unsigned seq)
          {
              HeapEntry &h = *new HeapEntry(adaptor, LINK(p), seq);
              IMessageResult *result = p->queryResult();
              assertex(result);
              if (h.noteResult(result->getCursor(rowManager), isCompleteMatchFlag(result)))
              {
                  heapEntries.append(h);
                  heap.append(heap.ordinality());
              }
              else
                  h.Release();
          }
  
          void removeHeap(unsigned idx)
          {
              heapEntries.remove(idx);
              ForEachItemIn(i, heap)
              {
                  unsigned v = heap.item(i);
                  assertex(v != idx);
                  if (v > idx)
                      heap.replace(v-1, i);
              }
          }
  
          bool isCompleteMatchFlag(IMessageResult *result)
          {
              unsigned metaLen;
              const byte *metaInfo = (const byte *) result->getMessageMetadata(metaLen);
              if (metaLen)
              {
                  unsigned continuationLen = *(unsigned *) metaInfo;
                  if (continuationLen >= sizeof(bool)*2)
                  {
                      metaInfo += sizeof(unsigned)+sizeof(bool);
                      return *(bool *) metaInfo;                     // The field we want is the second bool field in the continuation data, and is always stored uncompressed
                  }
              }
              return true; // if no continuation info, last row was complete.
          }
  
      public:
          CRemoteResultMerger(CRemoteResultAdaptor &_adaptor) : adaptor(_adaptor)
          {
              init(NULL, NULL);
          }
  
          void init(ISteppingMeta *meta, IRowManager *_rowManager)
          {
              if (meta)
              {
                  numFields = meta->getNumFields();
                  compare = meta->queryCompare();
              }
              else
              {
                  numFields = 0;
                  compare = NULL;
              }
              rowManager = _rowManager;
              numPending = 0;
              endSeen = false;
              remakePending = false;
              deferredContinuation = false;
          }
  
          void reset()
          {
              heapEntries.kill();
              heap.kill();
              numPending = 0;
              endSeen = false;
              remakePending = false;
              deferredContinuation = false;
          }
  
          inline bool noteEndSeen()
          {
              bool hadSeen = endSeen;
              if (!endSeen)
                  makeHeap();
              endSeen = true;
              return !hadSeen;
          }
  
          void noteResult(IRoxieServerQueryPacket *p, unsigned seq)
          {
              if (!p->isContinuation())
                  append(p, seq);
              else
              {
                  ForEachItemIn(idx, heapEntries)
                  {
                      HeapEntry &h = heapEntries.item(idx);
                      if (h.packet == p)
                      {
                          IMessageResult *result = p->queryResult();
                          if (!h.noteResult(result->getCursor(rowManager), isCompleteMatchFlag(result)))
                          {
                              heap.zap(idx);
                              removeHeap(idx);
                          }
                          numPending--;
                          if (!numPending)
                              makeHeap();
                          return;
                      }
                  }
              }
              // If we get here it must be a continuation for one that I have not yet consumed... we don't need to do anything.
              return;
          }
  
          unsigned skipRows(unsigned &idx, const void *seek, const void *rawSeek, unsigned numFields, unsigned seekLen, const SmartStepExtra & stepExtra)
          {
              HeapEntry &entry = heapEntries.item(idx);
              unsigned skipped = entry.current ? entry.skipTo(compare, seek, numFields, !stepExtra.returnMismatches()) : 0;
              if (!entry.current)
              {
                  IRoxieServerQueryPacket *continuation = entry.packet->queryContinuation();
                  if (continuation)
                  {
                      continuation->Link();
                      entry.packet->Release();
                      entry.packet = continuation;
                      if (continuation->hasResult())
                      {
                          IMessageResult *result = continuation->queryResult();
                          bool lastIsCompleteMatch = isCompleteMatchFlag(result);
                          entry.noteResult(result->getCursor(rowManager), lastIsCompleteMatch);
                      }
                      else
                      {
                          if (continuation->isDelayed())
                          {
                              continuation->setDelayed(false);
                              MemoryBuffer serializedSkip;
                              adaptor.activity.serializeSkipInfo(serializedSkip, seekLen, rawSeek, numFields, seek, stepExtra);
                              continuation->setPacket(continuation->queryPacket()->insertSkipData(serializedSkip.length(), serializedSkip.toByteArray()));
                              ROQ->sendPacket(continuation->queryPacket(), adaptor.activity.queryLogCtx());
                              adaptor.sentsome.signal();
                          }
                          numPending++;
                      }
                  }
                  else
                  {
                      heap.zap(idx);
                      removeHeap(idx);
                      idx--;
                  }
              }
              return skipped;
          }
  
          const void * nextRowGE(const void * seek, const void *rawSeek, unsigned numFields, unsigned seeklen, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
          {
              // We discard all rows < seekval from all entries in heap
              // If this results in additional agent requests, we return NULL so that we can wait for them
              // If not, we rebuild the heap (if any were skipped) and return the first row
              deferredContinuation = false;
              if (heap.length())
              {
                  unsigned skipped = 0;
                  unsigned idx = 0;
                  while(heapEntries.isItem(idx))
                  {
                      skipped += skipRows(idx, seek, rawSeek, numFields, seeklen, stepExtra);
                      idx++;
                  }
                  if (numPending)
                      return NULL;  // can't answer yet, need more results from agents
                  else
                  {
                      if (skipped)
                          makeHeap();
                      return next(wasCompleteMatch, stepExtra);
                  }
              }
              else
                  return NULL;
          }
  
          bool doContinuation(HeapEntry &topEntry, bool canDefer)
          {
              IRoxieServerQueryPacket *continuation = topEntry.packet->queryContinuation();
              if (continuation)
              {
                  if (continuation->isDelayed() && canDefer)
                  {
                      if (adaptor.activity.queryLogCtx().queryTraceLevel() > 10)
                          adaptor.activity.queryLogCtx().CTXLOG("Deferring continuation");
                      deferredContinuation = true;
                  }
                  else
                  {
                      deferredContinuation = false;
                      continuation->Link();
                      topEntry.packet->Release();
                      topEntry.packet = continuation;
                      if (continuation->hasResult())
                      {
                          IMessageResult *result = continuation->queryResult();
                          bool lastIsCompleteMatch = isCompleteMatchFlag(result);
                          topEntry.noteResult(result->getCursor(rowManager), lastIsCompleteMatch);
                      }
                      else
                      {
                          if (continuation->isDelayed()) // has the continuation been requested yet?
                          {
                              continuation->Link();
                              topEntry.packet->Release();
                              topEntry.packet = continuation;
                              continuation->setDelayed(false);
                              if (adaptor.activity.queryLogCtx().queryTraceLevel() > 10)
                                  adaptor.activity.queryLogCtx().CTXLOG("About to send continuation, from doContinuation");
                              ROQ->sendPacket(continuation->queryPacket(), adaptor.activity.queryLogCtx());
                              adaptor.sentsome.signal();
                          }
                          numPending++;   // we are waiting for one that is already in flight
                      }
                  }
                  return true;  // next not known yet
              }
              else
                  return false;
          }
  
          const void *next(bool & wasCompleteMatch, const SmartStepExtra & stepExtra)
          {
              OwnedConstRoxieRow ret;
              if (heap.length())
              {
                  if (deferredContinuation)
                  {
                      unsigned top = heap.item(0);
                      HeapEntry &topEntry = heapEntries.item(top);
                      doContinuation(topEntry, false);
                      return NULL;
                  }
                  if (remakePending)
                      remakeHeap();
                  unsigned top = heap.item(0);
                  HeapEntry &topEntry = heapEntries.item(top);
                  ret.set(topEntry.current);
                  wasCompleteMatch = topEntry.isCompleteMatch();
                  const void *next = topEntry.next();
                  if (!next)
                  {
                      if (!doContinuation(topEntry, stepExtra.returnMismatches()))
                      {
                          unsigned last = heap.popGet();
                          if (heap.length())
                              heap.replace(last, 0);
                          removeHeap(top);
                      }
                  }
                  remakePending = true;
              }
              return ret.getClear();
          }
  
          bool ready()
          {
              return endSeen && numPending == 0;  
          }
  
      };
  
      IRoxieServerQueryPacket *createRoxieServerQueryPacket(IRoxieQueryPacket *p)
      {
          return new CRoxieServerQueryPacket(p);
      }
  
  #ifdef _DEBUG
      void dumpPending()
      {
          CriticalBlock b(pendingCrit);
          ForEachItemIn(idx, pending)
          {
              IRoxieServerQueryPacket &p = pending.item(idx);
              StringBuffer s;
              unsigned __int64 dummy;
              if (p.isEnd())
                  s.append("END");
              else if (p.isLimit(dummy, dummy, dummy))
                  s.append("LIMIT");
              else
              {
                  IRoxieQueryPacket *i = p.queryPacket();
                  s.appendf("%s", p.hasResult() ? "COMPLETE " : "PENDING ");
                  if (i)
                  {
                      RoxiePacketHeader &header = i->queryHeader();
                      header.toString(s);
                  }
              }
              DBGLOG("Pending %d %s", idx, s.str());
          }
      }
  #endif
  
      void abortPending()
      {
          CriticalBlock b(pendingCrit);
          ForEachItemIn(idx, pending)
          {
              IRoxieServerQueryPacket &p = pending.item(idx);
              if (!p.hasResult())
              {
                  IRoxieQueryPacket *i = p.queryPacket();
                  if (i)
                  {
                      RoxiePacketHeader &header = i->queryHeader();
                      ROQ->sendAbort(header, activity.queryLogCtx());
                  }
              }
          }
          pending.kill();
      }
  
      void checkDelayed()
      {
          if (ctx->queryDebugContext() && ctx->queryDebugContext()->getExecuteSequentially())
          {
              bool allDelayed = true;
              CriticalBlock b(pendingCrit);
              unsigned sendIdx = (unsigned) -1;
              ForEachItemIn(idx, pending)
              {
                  IRoxieServerQueryPacket &p = pending.item(idx);
                  if (p.queryPacket())
                  {
                      if (p.isDelayed())
                      {
                          if (sendIdx == (unsigned) -1)
                              sendIdx = idx;
                      }
                      else if (!p.hasResult())
                      {
                          allDelayed = false;
                          break;
                      }
                  }
              }
              if (allDelayed && sendIdx != (unsigned) -1)
              {
                  if (activity.queryLogCtx().queryTraceLevel() > 10)
                      activity.queryLogCtx().CTXLOG("About to send debug-deferred from next");
                  pending.item(sendIdx).setDelayed(false);
                  ROQ->sendPacket(pending.item(sendIdx).queryPacket(), activity.queryLogCtx());
                  sentsome.signal();
              }
          }
          else if (deferredStart)
          {
              CriticalBlock b(pendingCrit);
              ForEachItemIn(idx, pending)
              {
                  IRoxieServerQueryPacket &p = pending.item(idx);
                  if (p.isDelayed())
                  {
                      if (activity.queryLogCtx().queryTraceLevel() > 10)
                          activity.queryLogCtx().CTXLOG("About to send deferred start from next");
                      p.setDelayed(false);
                      ROQ->sendPacket(p.queryPacket(), activity.queryLogCtx());
                      sentsome.signal();
                  }
              }
              deferredStart = false;
          }
      }
  
      void retryPending()
      {
          CriticalBlock b(pendingCrit);
          checkDelayed();
          ForEachItemIn(idx, pending)
          {
              IRoxieServerQueryPacket &p = pending.item(idx);
              if (!p.hasResult() && !p.isDelayed())
              {
                  IRoxieQueryPacket *i = p.queryPacket();
                  if (i)
                  {
                      if (!i->queryHeader().retry())
                      {
                          StringBuffer s;
                          IException *E = MakeStringException(ROXIE_MULTICAST_ERROR, "Failed to get response from agent(s) for %s in activity %d", i->queryHeader().toString(s).str(), activity.queryId());
                          activity.queryLogCtx().logOperatorException(E, __FILE__, __LINE__, "CRemoteResultAdaptor::retry");
                          throw E;
                      }
                      if (!localAgent) 
                      {
                          ROQ->sendPacket(i, activity.queryLogCtx());
                          retriesSent++;
                      }
                  }
              }
          }
      }
  
      class ChannelBuffer
      {
      protected:
          unsigned bufferLeft;
          MemoryBuffer buffer;
          char *nextBuf;
          unsigned overflowSequence;
          unsigned channel;  // == bonded channel
          bool needsFlush;
          InterruptableSemaphore flowController;
          const CRemoteResultAdaptor &owner;
          CriticalSection crit;
  
      public:
          ChannelBuffer(const CRemoteResultAdaptor &_owner, unsigned _channel) : channel(_channel), flowController(perChannelFlowLimit), owner(_owner)
          {
              overflowSequence = 0;
              needsFlush = false;
              bufferLeft = 0;
              nextBuf = NULL;
          }
  
          void init(unsigned minSize)
          {
              assertex(!buffer.length());
              if (minSize < MIN_PAYLOAD_SIZE)
                  minSize = MIN_PAYLOAD_SIZE;
              unsigned headerSize = sizeof(RoxiePacketHeader)+owner.headerLength();
              unsigned bufferSize = headerSize+minSize;
              if (bufferSize < mtu_size)
                  bufferSize = mtu_size;
              buffer.reserveTruncate(bufferSize);
              bufferLeft = bufferSize - headerSize;
              assertex(buffer.toByteArray());
              nextBuf = (char *) buffer.toByteArray() + headerSize;
              needsFlush = false;
          }
  
          inline IRoxieQueryPacket *flush()
          {
              CriticalBlock cb(crit);
              Owned<IRoxieQueryPacket> ret;
              if (needsFlush)
              {
                  buffer.setLength(nextBuf - buffer.toByteArray());
                  RoxiePacketHeader *h = (RoxiePacketHeader *) buffer.toByteArray();
                  h->init(owner.remoteId, owner.ruid, channel, overflowSequence);
  
                  //patch logPrefix, cachedContext and parent extract into the place reserved in the message buffer
                  byte * tgt = (byte*)(h+1);
                  owner.copyHeader(tgt, channel);
  
                  ret.setown(createRoxiePacket(buffer));
                  if (overflowSequence == OVERFLOWSEQUENCE_MAX)
                      overflowSequence = 1; // don't wrap to 0 - that is a bit special
                  else
                      overflowSequence++;
                  needsFlush = false;
                  bufferLeft = 0;
                  if (owner.flowControlled)
                  {
                      CriticalUnblock cub(crit);
                      while (!flowController.wait(1000))
                      {
                          StringBuffer s;
                          owner.activity.queryLogCtx().CTXLOG("Channel %d blocked by flow control: %s", channel, h->toString(s).str());
                      }
                  }
              }
              return ret.getClear();
          }
  
          inline void signal()
          {
              if (owner.flowControlled)
                  flowController.signal();
          }
  
          inline void interrupt(IException *e)
          {
              flowController.interrupt(e);
          }
  
          inline void *getBuffer(unsigned size)
          {
              CriticalBlock cb(crit);
              if (bufferLeft >= size)
              {
                  needsFlush = true;
                  void * ret = nextBuf;
                  nextBuf += size;
                  bufferLeft -= size;
                  return ret;
              }
              else if (!needsFlush)
              {
                  init(size);
                  return getBuffer(size);
              }
              else if (owner.mergeOrder)
              {
                  return buffer.reserve(size); // whole query needs to go as single packet if we are to merge
              }
              else
                  return NULL; // will force it to flush and start a new packet
          }
      };
  
  private:
      friend class CRemoteResultMerger;
      bool allread;
      bool contextCached;
      bool preserveOrder;
      bool eogSent = false;
  
      InterruptableSemaphore sentsome;
      Owned <IMessageCollator> mc;
      Owned<IMessageUnpackCursor> mu;
      Owned<IMessageResult> mr;
      ChannelBuffer **buffers;
      IHThorArg &helper;
      unsigned __int64 stopAfter;
      unsigned resendSequence;
      IHThorArg *colocalArg;
      IArrayOf<IRoxieServerQueryPacket> pending;
      CriticalSection pendingCrit;
      unsigned sentSequence;
      Owned<IOutputRowDeserializer> deserializer;
      Owned<IEngineRowAllocator> rowAllocator;
      CRemoteResultMerger merger;
  
      // this is only used to avoid recreating a bufferStream for each row.  A better solution may be needed
      MemoryBuffer tempRowBuffer;     
      Owned<ISerialStream> bufferStream;
      CThorStreamDeserializerSource rowSource;
  
  protected:
      IRowManager *rowManager;
      unsigned __int64 rowLimit;
      unsigned __int64 keyedLimit;
      IRoxieServerErrorHandler *errorHandler;
      CachedOutputMetaData meta;
  
  public:
      ISteppingMeta *mergeOrder;
      IRoxieAgentContext *ctx;
      IDebuggableContext *debugContext;
      IRoxieServerActivity &activity;
      unsigned parentExtractSize;
      const byte * parentExtract;
      bool flowControlled;
      bool deferredStart;
      MemoryBuffer logInfo;
      MemoryBuffer cachedContext;
      const RemoteActivityId &remoteId;
      ruid_t ruid;
      mutable CriticalSection buffersCrit;
      unsigned processed;
      cycle_t totalCycles;
      cycle_t unpackerWaitCycles;
      bool timeActivities;
  
  //private:   //vc6 doesn't like this being private yet accessed by nested class...
      const void *getRow(IMessageUnpackCursor *mu) 
      {
          if (!mu->isSerialized() || (meta.isFixedSize() && !deserializer))
              return mu->getNext(meta.getFixedSize());
          else
          {
              RecordLengthType *rowlen = (RecordLengthType *) mu->getNext(sizeof(RecordLengthType));
              if (rowlen)
              {
                  RecordLengthType len = *rowlen;
                  ReleaseRoxieRow(rowlen);
                  const void *agentRec = mu->getNext(len);
                  if (deserializer && mu->isSerialized())
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      tempRowBuffer.setBuffer(len, const_cast<void *>(agentRec), false);
                      size_t outsize = deserializer->deserialize(rowBuilder, rowSource);
                      ReleaseRoxieRow(agentRec);
                      return rowBuilder.finalizeRowClear(outsize);
                  }
                  else
                      return agentRec;
              }
              else
                  return NULL;
          }
      }
  
  private:
  
      ChannelBuffer *queryChannelBuffer(unsigned channel, bool force=false)
      {
          CriticalBlock cb(buffersCrit);
          ChannelBuffer *b = buffers[channel];
          if (!b && force)
          {
              if (!contextCached)
              {
                  logInfo.clear();
                  unsigned char loggingFlags = LOGGING_FLAGSPRESENT | LOGGING_TRACELEVELSET;
                  unsigned char ctxTraceLevel = activity.queryLogCtx().queryTraceLevel() + 1; // Avoid passing a 0
                  const QueryOptions &options = ctx->queryOptions();
                  if (activity.queryLogCtx().isIntercepted())
                      loggingFlags |= LOGGING_INTERCEPTED;
                  if (options.timeActivities)
                      loggingFlags |= LOGGING_TIMEACTIVITIES; 
                  if (activity.queryLogCtx().isBlind())
                      loggingFlags |= LOGGING_BLIND;
                  if (options.checkingHeap)
                      loggingFlags |= LOGGING_CHECKINGHEAP;
                  if (ctx->queryWorkUnit())
                      loggingFlags |= LOGGING_WUID;
                  if (debugContext)
                  {
                      loggingFlags |= LOGGING_DEBUGGERACTIVE;
                      logInfo.append(loggingFlags).append(ctxTraceLevel);
                      MemoryBuffer bpInfo;
                      debugContext->serialize(bpInfo);
                      bpInfo.append((__uint64)(memsize_t) &activity);
                      logInfo.append((unsigned short) bpInfo.length());
                      logInfo.append(bpInfo.length(), bpInfo.toByteArray());
                  }
                  else
                      logInfo.append(loggingFlags).append(ctxTraceLevel);
                  StringBuffer logPrefix;
                  activity.queryLogCtx().getLogPrefix(logPrefix);
                  logInfo.append(logPrefix);
                  activity.serializeCreateStartContext(cachedContext.clear());
                  activity.serializeExtra(cachedContext);
                  if (activity.queryVarFileInfo())
                  {
                      activity.queryVarFileInfo()->queryTimeStamp().serialize(cachedContext);
                      cachedContext.append(activity.queryVarFileInfo()->queryCheckSum());
                  }
                  contextCached = true;
              }
  
              b = buffers[channel] = new ChannelBuffer(*this, channel);
          }
          return b;
      }
  
      void processRow(const void *got)
      {
          processed++;
          if (processed > rowLimit)
          {
              ReleaseRoxieRow(got);
              errorHandler->onLimitExceeded(false); // NOTE - should throw exception
              throwUnexpected();
          }
          else if (processed > keyedLimit)
          {
              ReleaseRoxieRow(got);
              errorHandler->onLimitExceeded(true); // NOTE - should throw exception
              throwUnexpected();
          }
      }
  
  public:
      IMPLEMENT_IINTERFACE;
  
      CRemoteResultAdaptor(IRoxieAgentContext *_ctx, IRoxieServerErrorHandler *_errorHandler, const RemoteActivityId &_remoteId, IOutputMetaData *_meta, IHThorArg &_helper, IRoxieServerActivity &_activity, bool _preserveOrder, bool _flowControlled)
          : preserveOrder(_preserveOrder), helper(_helper), merger(*this), meta(_meta), activity(_activity), flowControlled(_flowControlled), remoteId(_remoteId)
      {
          ctx = _ctx;
          errorHandler = _errorHandler;
          colocalArg = NULL;
          allread = false;
          ruid = 0;
          rowLimit = (unsigned __int64) -1;
          keyedLimit = (unsigned __int64) -1;
          contextCached = false;
          stopAfter = I64C(0x7FFFFFFFFFFFFFFF);
          buffers = new ChannelBuffer*[numChannels+1];
          memset(buffers, 0, (numChannels+1)*sizeof(ChannelBuffer *));
          parentExtractSize = 0;
          rowManager = NULL;
          parentExtract = NULL;
          debugContext = NULL;
          mergeOrder = NULL;
          deferredStart = false;
          processed = 0;
          sentSequence = 0;
          resendSequence = 0;
          totalCycles = 0;
          unpackerWaitCycles = 0;
          bufferStream.setown(createMemoryBufferSerialStream(tempRowBuffer));
          rowSource.setStream(bufferStream);
          timeActivities = defaultTimeActivities;
      }
  
      ~CRemoteResultAdaptor()
      {
          if (mc)
          {
              ROQ->queryReceiveManager()->detachCollator(mc);
              mc.clear();
          }
          for (unsigned channel = 0; channel <= numChannels; channel++)
          {
              delete(buffers[channel]);
          }
          delete [] buffers;
      }
  
      void setMeta(IOutputMetaData *newmeta)
      {
          meta.set(newmeta);
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
  
      virtual IRoxieServerActivity *queryActivity()
      {
          return &activity;
      }
  
      virtual IIndexReadActivityInfo *queryIndexReadActivity()
      {
          return NULL;
      }
  
      void setMergeInfo(ISteppingMeta *_mergeOrder)
      {
          mergeOrder = _mergeOrder;
          deferredStart = true;
      }
  
      void send(IRoxieQueryPacket *p)
      {
          if (p)
          {
              Linked<IRoxieQueryPacket> saver(p); // avoids a race with abortPending, without keeping pendingCrit locked over the send which we might prefer not to
              assertex(p->queryHeader().uid==ruid);
  
              // MORE: Maybe we should base the fastlane flag on some other 
              //       criteria !! (i.e A Roxie server prediction based on the 
              //       activity type/activity behaviour/expected reply size .. etc).
              //       
              //       Currently (code below) based on high priority, seq=0, and none-child activity.
              //       But this could still cause too many reply packets on the fastlane
              //       (higher priority output Q), which may cause the activities on the 
              //       low priority output Q to not get service on time.
              if ((colocalArg == 0) &&     // not a child query activity??
                      (p->queryHeader().activityId & (ROXIE_SLA_PRIORITY | ROXIE_HIGH_PRIORITY)) && 
                      (p->queryHeader().overflowSequence == 0) &&
                      (p->queryHeader().continueSequence & ~CONTINUE_SEQUENCE_SKIPTO)==0)
                  p->queryHeader().retries |= ROXIE_FASTLANE;
  
              if (p->queryHeader().channel)
              {
                  IRoxieServerQueryPacket *rsqp = createRoxieServerQueryPacket(p);
                  if (deferredStart)
                      rsqp->setDelayed(true);
                  rsqp->setSequence(sentSequence++);
                  {
                      CriticalBlock b(pendingCrit);
                      pending.append(*rsqp);
                  }
                  if (!deferredStart)
                  {
                      ROQ->sendPacket(p, activity.queryLogCtx());
                      sentsome.signal();
                  }
              }
              else
              {
                  // Care is needed here. If I send the packet before I add to the pending there is a danger that I'll get results that I discard 
                  // Need to add first, then send
                  unsigned i;
                  for (i = 1; i <= numChannels; i++)
                  {
                      IRoxieQueryPacket *q = p->clonePacket(i);
                      IRoxieServerQueryPacket *rsqp = createRoxieServerQueryPacket(q);
                      rsqp->setSequence(sentSequence++);
                      if (deferredStart)
                      {
                          rsqp->setDelayed(true);
                      }
                      {
                          CriticalBlock b(pendingCrit);
                          pending.append(*rsqp);
                      }
                      if (!deferredStart)
                              sentsome.signal();
                  }
                  if (!deferredStart)
                      ROQ->sendPacket(p, activity.queryLogCtx());
                  buffers[0]->signal(); // since replies won't come back on that channel...
                  p->Release();
              }
          }
      }
  
      void *getMem(unsigned partNo, unsigned fileNo, unsigned size)
      {
          unsigned channel = partNo ? getBondedChannel(partNo) : 0;
          size += sizeof(PartNoType);
          ChannelBuffer *b = queryChannelBuffer(channel, true);
          char *buffer = (char *) b->getBuffer(size);
          if (!buffer)
          {
              send(b->flush());
              buffer = (char *) b->getBuffer(size);
          }
          PartNoType sp;
          sp.partNo = partNo;
          sp.fileNo = fileNo;
          memcpy(buffer, &sp, sizeof(sp));
          buffer += sizeof(sp);
          return buffer;
      }
  
      void injectResult(IMessageResult *result)
      {
          IRoxieServerQueryPacket *f = new CRoxieServerQueryPacket(NULL);
          f->setSequence(sentSequence++);
          f->setResult(result);
          CriticalBlock b(pendingCrit);
          pending.append(*f);
          sentsome.signal(); // MORE - arguably should only send if there is any point waking up the listener thread, to save context swicth
      }
  
      void flush()
      {
          for (unsigned channel = 0; channel <= numChannels; channel++)
          {
              ChannelBuffer *b = queryChannelBuffer(channel, false);
              if (b)
                  send(b->flush());
          }
      }
  
      void interruptBuffers(IException *e)
      {
          for (unsigned channel = 0; channel <= numChannels; channel++)
          {
              ChannelBuffer *b = queryChannelBuffer(channel, false);
              if (b)
                  b->interrupt(LINK(e));
          }
      }
  
      void senddone()
      {
          CriticalBlock b(pendingCrit);
          pending.append(*new CRoxieServerQueryPacketEndMarker);
          sentsome.signal();
      }
  
      bool fireException(IException *e)
      {
          {
              CriticalBlock b(pendingCrit);
              pending.append(*new CRoxieServerQueryPacketEndMarker);
          }
          interruptBuffers(e);
          if (mc)
              mc->interrupt(LINK(e));
          sentsome.interrupt(e);
          return true;
      }
  
      virtual void onCreate(IHThorArg *_colocalArg)
      {
          debugContext = ctx->queryDebugContext();
          colocalArg = _colocalArg;
          if (meta.needsSerializeDisk())
          {
              deserializer.setown(meta.createDiskDeserializer(ctx->queryCodeContext(), activity.queryId()));
              rowAllocator.setown(activity.createRowAllocator(meta.queryOriginal()));
          }
          if (ctx->queryDebugContext() && ctx->queryDebugContext()->getExecuteSequentially())
              deferredStart = true;
          timeActivities = ctx->queryOptions().timeActivities;
      }
  
      virtual void onStart(unsigned _parentExtractSize, const byte * _parentExtract)
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAonstart");
  #endif
          sentsome.reinit();
          ruid = getNextRuid();
          rowManager = &ctx->queryRowManager();
          if (mergeOrder)
              merger.init(mergeOrder, rowManager);
          if (mc)
          {
              ROQ->queryReceiveManager()->detachCollator(mc); // Should never happen - implies someone forgot to call onReset!
          }
          mc.setown(ROQ->queryReceiveManager()->createMessageCollator(rowManager, ruid));
          allread = false;
          mu.clear();
          contextCached = false;
          processed = 0;
          resendSequence = 0;
          sentSequence = 0;
          for (unsigned channel = 0; channel <= numChannels; channel++)
          {
              delete(buffers[channel]);
              buffers[channel] = NULL;
          }
          flush();
          parentExtractSize = _parentExtractSize;
          parentExtract = _parentExtract;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAstart");
  #endif
          activity.start(parentExtractSize, parentExtract, paused);
      }
  
      void setLimits(unsigned __int64 _rowLimit, unsigned __int64 _keyedLimit, unsigned __int64 _stopAfter)
      {
          if (ctx->queryProbeManager())
          {
              if (_rowLimit != (unsigned __int64) -1) ctx->queryProbeManager()->setNodePropertyInt(&activity, "rowLimit", _rowLimit);
              if (_keyedLimit != (unsigned __int64) -1) ctx->queryProbeManager()->setNodePropertyInt(&activity, "keyedLimit", _keyedLimit);
              if (_stopAfter != I64C(0x7FFFFFFFFFFFFFFF)) ctx->queryProbeManager()->setNodePropertyInt(&activity, "choosenLimit", _stopAfter);
          }
          {
              CriticalBlock b(pendingCrit);
              if (pending.length())
              {
  #ifdef _DEBUG
                  dumpPending();   // MORE - only defined in debug build - could have put the ifdef inside the dumpPending method
  #endif
                  assertex(pending.length()==0);
              }
              pending.append(*new CRoxieServerQueryPacketLimitMarker(_rowLimit, _keyedLimit, _stopAfter));
          }
          sentsome.signal();
          rowLimit = _rowLimit;
          keyedLimit = _keyedLimit;
          stopAfter = _stopAfter;
      }
  
      virtual void stop()
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAstop");
  #endif
          onStop();
          activity.stop();
      }
  
      void onStop()
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAonstop");
  #endif
          abortPending();
          interruptBuffers(NULL);
          sentsome.interrupt();
          if (mc)  // May not be set if start() chain threw exception
              mc->interrupt();
      }
  
      virtual void reset()
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAreset");
  #endif
          activity.reset();
          onReset();
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          // Make sure parent activity calls up the tree
          PointerArrayOf<IEngineRowStream> instreams;
          IStrandJunction *junction = activity.getOutputStreams(ctx, idx, instreams, consumerOptions, consumerOrdered, orderedCallbacks);
          assertex (junction == NULL);
          // I return a single strand (we could change that sometime I guess...)
          streams.append(this);
          return NULL;
      }
  
      virtual void resetEOF()
      {
          throwUnexpected();
      }
  
      virtual void onReset()
      {
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
              activity.queryLogCtx().CTXLOG("RRAonreset");
  #endif
          if (mc)
              ROQ->queryReceiveManager()->detachCollator(mc);
          merger.reset();
          pending.kill();
          if (mc)
          {
              activity.noteStatistic(StSizeAgentReply, mc->queryBytesReceived());
              activity.noteStatistic(StTimeAgentWait, cycle_to_nanosec(unpackerWaitCycles));
              if (ctx)
                  ctx->addAgentsReplyLen(mc->queryBytesReceived(), mc->queryDuplicates(), mc->queryResends());
          }
          mc.clear(); // Or we won't free memory for graphs that get recreated
          mu.clear(); //ditto
          deferredStart = false;
          // NOTE: do NOT clear mergeOrder - this is set at create time not per child query
      }
  
      virtual IOutputMetaData * queryOutputMeta() const
      {
          return helper.queryOutputMeta();
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return totalCycles;
      }
  
      void gatherStats(CRuntimeStatisticCollection & merged) const
      {
          if (rowAllocator)
              rowAllocator->gatherStats(merged);
      }
  
      const void * nextRowGE(const void *seek, const void *rawSeek, unsigned numFields, unsigned seekLen, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          if (activity.queryLogCtx().queryTraceLevel() > 20)
          {
              StringBuffer recstr;
              unsigned i;
              for (i = 0; i < seekLen; i++)
              {
                  recstr.appendf("%02x ", ((unsigned char *) rawSeek)[i]);
              }
              activity.queryLogCtx().CTXLOG("CRemoteResultAdaptor::nextRowGE(rawSeek=%s numFields=%d, seeklen=%d, returnMismatches=%d)", recstr.str(), numFields, seekLen, stepExtra.returnMismatches());
          }
          assertex(mergeOrder);
          if (deferredStart)
          {
              CriticalBlock b(pendingCrit);
              ForEachItemIn(idx, pending)
              {
                  IRoxieServerQueryPacket &p = pending.item(idx);
                  if (p.isDelayed())
                  {
                      p.setDelayed(false);
                      if (activity.queryLogCtx().queryTraceLevel() > 10)
                          activity.queryLogCtx().CTXLOG("About to send deferred start from nextRowGE, setting requireExact to %d", !stepExtra.returnMismatches());
                      MemoryBuffer serializedSkip;
                      activity.serializeSkipInfo(serializedSkip, seekLen, rawSeek, numFields, seek, stepExtra);
                      p.setPacket(p.queryPacket()->insertSkipData(serializedSkip.length(), serializedSkip.toByteArray()));
                      ROQ->sendPacket(p.queryPacket(), activity.queryLogCtx());
                      sentsome.signal();
                  }
              }
              deferredStart = false;
          }
          SimpleActivityTimer t(totalCycles, timeActivities);
          if (processed==stopAfter)
              return NULL;
          if (allread)
              return NULL;
          for (;;)
          {
              if (merger.ready())
              {
                  const void *got = merger.nextRowGE(seek, rawSeek, numFields, seekLen, wasCompleteMatch, stepExtra);
                  if (got)
                  {
                      processRow(got);
                      return got;
                  }
              }
              if (!reload()) // MORE - should pass the seek info here...
                  return NULL;
          }
      }
  
      virtual const void *nextRow()
      {
          // If we are merging then we need to do a heapsort on all 
          SimpleActivityTimer t(totalCycles, timeActivities);
          if (activity.queryLogCtx().queryTraceLevel() > 10)
          {
              activity.queryLogCtx().CTXLOG("CRemoteResultAdaptor::nextRow()");
          }
          for (;;)
          {
              checkDelayed();
              if (processed==stopAfter)
                  return NULL;
              if (allread)
                  return NULL;
              // If we can still consume from the merger or the most recently retrieved mu, do so. 
              const void *got = NULL;
              if (mergeOrder && merger.ready())
              {
                  bool matched = true;
                  got = merger.next(matched, dummySmartStepExtra);
              }
              else if (mu)
              {
                  got = getRow(mu);
                  if (!got && meta.isGrouped() && !eogSent)
                  {
                      eogSent = true;
                      return NULL;
                  }
              }
              if (got)
              {
                  processRow(got);
                  return got;
              }
              eogSent = false;
              if (!reload())
                  return NULL;
          }
      }
  
      bool reload()
      {
          // Wait for something to be returned from a agent....
          mu.clear();
          sentsome.wait();
          // must be at least an endMarker on the queue since sentsome was signalled
          { 
              CriticalBlock b(pendingCrit);
              IRoxieServerQueryPacket &top = pending.item(0);
              if (top.isLimit(rowLimit, keyedLimit, stopAfter)) // This is really a start marker...
              {
                  pending.remove(0);
                  return true;
              }
              else if (top.isEnd())
              {
                  pending.remove(0);
                  allread = true;
                  if (activity.queryLogCtx().queryTraceLevel() > 5)
                      activity.queryLogCtx().CTXLOG("All read on ruid %x", ruid);
                  return false;
              }
              else if (mergeOrder)
              {
                  unsigned idx = 0;
                  bool added = false;
                  while (pending.isItem(idx))
                  {
                      IRoxieServerQueryPacket &item = pending.item(idx);
                      if (item.isEnd())
                      {
                          if (merger.noteEndSeen())
                          {
                              sentsome.signal(); // Because we waited, yet didn't actually consume anything
                              added = true;
                          }
                          break;
                      }
                      else if (item.hasResult())
                      {
                          merger.noteResult(&item, item.getSequence());
                          pending.remove(idx);
                          added = true;
                      }
                      else if (item.isContinuation())
                          idx++;
                      else
                          break;
                  }
                  if (added)
                      return true;
              }
              else if (top.hasResult())
              {
                  mr.setown(pending.item(0).getResult());
                  mu.setown(mr->getCursor(rowManager));
                  pending.remove(0);
                  return true;
              }
          }
          getNextUnpacker();
          return true;
      }
  
      void getNextUnpacker()
      {
          mu.clear();
          SimpleActivityTimer t(unpackerWaitCycles, timeActivities);
          unsigned ctxTraceLevel = activity.queryLogCtx().queryTraceLevel();
          unsigned timeout = remoteId.isSLAPriority() ? slaTimeout : (remoteId.isHighPriority() ? highTimeout : lowTimeout);
          unsigned checkInterval = activity.queryContext()->checkInterval();
          if (checkInterval > timeout)
              checkInterval = timeout;
          unsigned lastActivity = msTick();
          for (;;)
          {
              checkDelayed();
              activity.queryContext()->checkAbort();
              bool anyActivity;
              if (ctxTraceLevel > 5)
                  activity.queryLogCtx().CTXLOG("Calling getNextUnpacker(%d)", checkInterval);
              mr.setown(mc->getNextResult(checkInterval, anyActivity));
              if (ctxTraceLevel > 6)
                  activity.queryLogCtx().CTXLOG("Called getNextUnpacker(%d), activity=%d", checkInterval, anyActivity);
              activity.queryContext()->checkAbort();
              if (anyActivity)
                  lastActivity = msTick();
              if (mr)
              {
                  unsigned roxieHeaderLen;
                  const RoxiePacketHeader &header = *(const RoxiePacketHeader *) mr->getMessageHeader(roxieHeaderLen);
  #ifdef _DEBUG
                  assertex(roxieHeaderLen == sizeof(RoxiePacketHeader));
  #endif
                  if (ctxTraceLevel > 5)
                  {
                      StringBuffer s;
                      activity.queryLogCtx().CTXLOG("getNextUnpacker got packet %s", header.toString(s).str());
                  }
  
                  CriticalBlock b(pendingCrit);
                  unsigned idx = 0;
                  IRoxieServerQueryPacket *original = NULL;
                  IRoxieQueryPacket *op;
                  while (pending.isItem(idx))
                  {
                      original = &pending.item(idx);
                      op = original->queryPacket();
                      if (op && header.matchPacket(op->queryHeader()))
                          break;
                      original = NULL;
                      idx++;
                  }
                  if (!original || original->hasResult())
                  {
                      switch (header.activityId)
                      {
                          case ROXIE_FILECALLBACK:
                          {
                              // tell agent to abort 
                              //if (ctxTraceLevel > 5)
                              {
                                  StringBuffer s;
                                  activity.queryLogCtx().CTXLOG("Redundant callback on query %s", header.toString(s).str());
                              }
                              Owned<IMessageUnpackCursor> callbackData = mr->getCursor(rowManager);
                              OwnedConstRoxieRow len = callbackData->getNext(sizeof(RecordLengthType));
                              if (len)
                              {
                                  RecordLengthType *rowlen = (RecordLengthType *) len.get();
                                  OwnedConstRoxieRow row = callbackData->getNext(*rowlen);
                                  const char *rowdata = (const char *) row.get();
                                  // bool isOpt = * (bool *) rowdata;
                                  // bool isLocal = * (bool *) (rowdata+1);
                                  ROQ->sendAbortCallback(header, rowdata+2, activity.queryLogCtx());
                              }
                              else
                                  throwUnexpected();
                              break;
                          }
                          // MORE - ROXIE_ALIVE perhaps should go here too
                          default:
                              if (ctxTraceLevel > 3)
                                  activity.queryLogCtx().CTXLOG("Discarding packet %p %x - original %p is NULL or has result already", mr.get(), header.activityId, original);
                              mr->discard();
                              break;
                      }
                      mr.clear();
                  }
                  else 
                  {
                      resultsReceived++;
                      switch (header.activityId)
                      {
                      case ROXIE_DEBUGCALLBACK:
                          {
                          Owned<IMessageUnpackCursor> callbackData = mr->getCursor(rowManager);
                          OwnedConstRoxieRow len = callbackData->getNext(sizeof(RecordLengthType));
                          if (len)
                          {
                              RecordLengthType *rowlen = (RecordLengthType *) len.get();
                              OwnedConstRoxieRow row = callbackData->getNext(*rowlen);
                              char *rowdata = (char *) row.get();
                              if (ctxTraceLevel > 5)
                              {
                                  StringBuffer s;
                                  activity.queryLogCtx().CTXLOG("Callback on query %s for debug", header.toString(s).str());
                              }
                              MemoryBuffer agentInfo;
                              agentInfo.setBuffer(*rowlen, rowdata, false);
                              unsigned debugSequence;
                              agentInfo.read(debugSequence);
                              Owned<IRoxieQueryPacket> reply = original->getDebugResponse(debugSequence);
                              if (!reply)
                                  reply.setown(activity.queryContext()->queryDebugContext()->onDebugCallback(header, *rowlen, rowdata));
                              if (reply)
                              {
                                  original->setDebugResponse(debugSequence, reply);
                                  ROQ->sendPacket(reply, activity.queryLogCtx());
                              }
  
                          }
                          else
                              throwUnexpected();
                          // MORE - somehow we need to make sure agent gets a reply even if I'm not waiting (in udp layer)
                          // Leave original message on pending queue in original location - this is not a reply to it.
                          break;
                          }
  
                      case ROXIE_FILECALLBACK:
                          {
                          // we need to send back to the agent a message containing the file info requested.
                          Owned<IMessageUnpackCursor> callbackData = mr->getCursor(rowManager);
                          OwnedConstRoxieRow len = callbackData->getNext(sizeof(RecordLengthType));
                          if (len)
                          {
                              RecordLengthType *rowlen = (RecordLengthType *) len.get();
                              OwnedConstRoxieRow row = callbackData->getNext(*rowlen);
                              const char *rowdata = (const char *) row.get();
                              bool isOpt = * (bool *) rowdata;
                              bool isLocal = * (bool *) (rowdata+1);
                              const char *lfn = rowdata+2;
                              if (ctxTraceLevel > 5)
                              {
                                  StringBuffer s;
                                  activity.queryLogCtx().CTXLOG("Callback on query %s file %s", header.toString(s).str(),(const char *) lfn);
                              }
                              activity.queryContext()->onFileCallback(header, lfn, isOpt, isLocal, defaultPrivilegedUser);
                          }
                          else
                              throwUnexpected();
                          // MORE - somehow we need to make sure agent gets a reply even if I'm not waiting (in udp layer)
                          // Leave original message on pending queue in original location - this is not a reply to it.
                          break;
                          }
                      case ROXIE_KEYEDLIMIT_EXCEEDED:
                          activity.queryLogCtx().CTXLOG("ROXIE_KEYEDLIMIT_EXCEEDED");
                          errorHandler->onLimitExceeded(true);    // NOTE - should throw exception!
                          throwUnexpected();
  
                      case ROXIE_LIMIT_EXCEEDED:
                          activity.queryLogCtx().CTXLOG("ROXIE_LIMIT_EXCEEDED");
                          errorHandler->onLimitExceeded(false);   // NOTE - should throw exception!
                          throwUnexpected();
  
                      case ROXIE_TRACEINFO:
                      {
                          Owned<IMessageUnpackCursor> extra = mr->getCursor(rowManager);
                          for (;;)
                          {
                              RecordLengthType *rowlen = (RecordLengthType *) extra->getNext(sizeof(RecordLengthType));
                              if (rowlen)
                              {
                                  char *logInfo = (char *) extra->getNext(*rowlen);
                                  MemoryBuffer buf;
                                  buf.setBuffer(*rowlen, logInfo, false);
                                  switch ((TracingCategory) *logInfo)
                                  {
                                  case LOG_TRACING:
                                  case LOG_ERROR:
                                      activity.queryLogCtx().CTXLOGl(new LogItem(buf));
                                      break;
                                  case LOG_STATVALUES:
                                      buf.skip(1);
                                      activity.mergeStats(buf);
                                      break;
                                  case LOG_CHILDCOUNT:
                                  case LOG_CHILDSTATS:
                                      //These need rethinking - all child stats should be serialized in a single block, and should be merged into
                                      //the information for an activity, ready for reporting later
                                      unsigned graphId, childId;
                                      buf.skip(1);
                                      buf.read(graphId);
                                      buf.read(childId);
                                      if (*logInfo == LOG_CHILDCOUNT)
                                      {
                                          unsigned idx;
                                          unsigned childProcessed;
                                          unsigned childStrands;
                                          buf.read(idx);
                                          buf.read(childProcessed);
                                          buf.read(childStrands);
                                          if (traceLevel > 5)
                                              activity.queryLogCtx().CTXLOG("Processing ChildCount %d idx %d strands %d for child %d subgraph %d", childProcessed, idx, childStrands, childId, graphId);
                                          //activity.queryContext()->noteProcessed(graphId, childId, idx, childProcessed, childStrands);
                                      }
                                      else
                                      {
                                          CRuntimeStatisticCollection childStats(allStatistics);
                                          childStats.deserialize(buf);
                                          if (traceLevel > 5)
                                          {
                                              StringBuffer s;
                                              activity.queryLogCtx().CTXLOG("Processing ChildStats for child %d subgraph %d: %s", childId, graphId, childStats.toStr(s).str());
                                          }
                                          //activity.queryContext()->mergeActivityStats(childStats, graphId, childId);
                                          activity.mergeStats(childStats);
                                      }
                                  }
                                  ReleaseRoxieRow(rowlen);
                                  ReleaseRoxieRow(logInfo);
                              }
                              else
                                  break;
                          }
                          break;
                      }
                      case ROXIE_EXCEPTION:
                          if (ctxTraceLevel > 1)
                          {
                              StringBuffer s;
                              activity.queryLogCtx().CTXLOG("Exception on query %s", header.toString(s).str());
                          }
                          op->queryHeader().noteException(header.retries);
                          if (op->queryHeader().allChannelsFailed())
                          {
                              activity.queryLogCtx().CTXLOG("Multiple exceptions on query - aborting");
                              Owned<IMessageUnpackCursor> exceptionData = mr->getCursor(rowManager);
                              throwRemoteException(exceptionData);
                          }
                          // Leave it on pending queue in original location
                          break;
  
                      case ROXIE_ALIVE:
                          if (ctxTraceLevel > 4)
                          {
                              StringBuffer s;
                              activity.queryLogCtx().CTXLOG("ROXIE_ALIVE: %s", header.toString(s).str());
                          }
                          op->queryHeader().noteAlive(header.retries & ROXIE_RETRIES_MASK);
                          // Leave it on pending queue in original location
                          break;
  
                      default:
                          if (header.retries & ROXIE_RETRIES_MASK)
                              retriesNeeded++;
                          unsigned metaLen;
                          const void *metaData = mr->getMessageMetadata(metaLen);
                          if (metaLen)
                          {
                              // We got back first chunk but there is more.
                              // resend the packet, with the cursor info provided.
                              // MORE - if smart-stepping, we don't want to send the continuation immediately. Other cases it's not clear that we do.
                              if (ctxTraceLevel > 1)
                              {
                                  StringBuffer s;
                                  activity.queryLogCtx().CTXLOG("Additional data size %d on query %s mergeOrder %p", metaLen, header.toString(s).str(), mergeOrder);
                              }
                              if (*((unsigned *) metaData) + sizeof(unsigned) != metaLen)
                              {
                                  StringBuffer s;
                                  activity.queryLogCtx().CTXLOG("Additional data size %d on query %s mergeOrder %p", metaLen, header.toString(s).str(), mergeOrder);
                                  activity.queryLogCtx().CTXLOG("Additional data is corrupt");
                                  throwUnexpected();
                              }
  
                              MemoryBuffer nextQuery;
                              nextQuery.append(sizeof(RoxiePacketHeader), &header);
                              nextQuery.append(op->getTraceLength(), op->queryTraceInfo());
                              nextQuery.append(metaLen, metaData);
                              nextQuery.append(op->getContextLength(), op->queryContextData());
                              if (resendSequence == CONTINUESEQUENCE_MAX)
                              {
                                  activity.queryLogCtx().CTXLOG("ERROR: Continuation sequence wrapped"); // shouldn't actually matter.... but suggests a very iffy query!
                                  resendSequence = 1;
                              }
                              else
                                  resendSequence++;
                              RoxiePacketHeader *newHeader = (RoxiePacketHeader *) nextQuery.toByteArray();
                              newHeader->continueSequence = resendSequence; // NOTE - we clear the skipTo flag since continuation of a skip is NOT a skip...
                              newHeader->retries &= ~ROXIE_RETRIES_MASK;
                              IRoxieQueryPacket *resend = createRoxiePacket(nextQuery);
                              CRoxieServerQueryPacket *fqp = new CRoxieServerQueryPacket(resend);
                              fqp->setSequence(original->getSequence());
                              pending.add(*fqp, idx+1); // note that pending takes ownership. sendPacket does not release.
                              original->setContinuation(LINK(fqp));
                              if (mergeOrder)
                                  fqp->setDelayed(true);
                              else
                              {
                                  ROQ->sendPacket(resend, activity.queryLogCtx());
                                  sentsome.signal();
                              }
                          }
                          unsigned channel = header.channel;
                          {
                              ChannelBuffer *b = queryChannelBuffer(channel); // If not something is wrong, or we sent out on channel 0?
                              if (b)
                                  b->signal();
                          }
                          original->setResult(mr.getClear());
                          sentsome.signal();
                          return;
                      }
                  }
              }
              else
              {
                  unsigned timeNow = msTick();
                  if (!anyActivity && !localAgent && (timeNow-lastActivity >= timeout))
                  {
                      lastActivity = timeNow;
                      activity.queryLogCtx().CTXLOG("Input has stalled for %u ms - retry required?", timeout);
                      retryPending();
                  }
              }
          }
      }
  
      inline unsigned headerLength() const
      {
          return logInfo.length() + cachedContext.length() + sizeof(unsigned) + parentExtractSize;
      }
  
      void copyHeader(byte *tgt, unsigned channel) const
      {
          unsigned len = logInfo.length();
          memcpy(tgt, logInfo.toByteArray(), len);
          tgt += len;
          *(unsigned *) tgt = parentExtractSize;
          tgt += sizeof(unsigned);
          if (parentExtractSize)
          {
              memcpy(tgt, parentExtract, parentExtractSize);
              tgt += parentExtractSize;
          }
          if (cachedContext.length())
          {
              memcpy(tgt, cachedContext.toByteArray(), cachedContext.length());
              tgt += cachedContext.length();
          }
      }
  };
  
  class CSkippableRemoteResultAdaptor : public CRemoteResultAdaptor
  {
      Owned <IException> exception;
      bool skipping;
      ConstPointerArray buff;
      unsigned index;
      bool pulled;
  
      void pullInput()
      {
          try
          {
              if (exception)
                  throw exception.getClear();
              unsigned __int64 count = 0;
              for (;;)
              {
                  const void * next = CRemoteResultAdaptor::nextRow();
                  if (next == NULL)
                  {
                      next = CRemoteResultAdaptor::nextRow();
                      if(next == NULL)
                          break;
                      buff.append(NULL);
                  }
                  count++;
                  if (count > rowLimit)
                  {
                      ReleaseRoxieRow(next);
                      ReleaseRoxieRows(buff);
                      errorHandler->onLimitExceeded(false); // throws an exception - user or LimitSkipException
                      throwUnexpected();
                  }
                  else if (count > keyedLimit)
                  {
                      ReleaseRoxieRow(next);
                      ReleaseRoxieRows(buff);
                      errorHandler->onLimitExceeded(true); // throws an exception - user or LimitSkipException
                      throwUnexpected();
                  }
                  buff.append(next);
              }
          }
          catch (IException *E)
          {
              if (QUERYINTERFACE(E, LimitSkipException))
              {
                  Owned<IException> cleanup = E;
                  ReleaseRoxieRows(buff);
                  const void *onfail = errorHandler->createLimitFailRow(E->errorCode() == KeyedLimitSkipErrorCode);
                  if (onfail)
                      buff.append(onfail);
              }
              else
                  throw;
          }
          pulled = true;
      }
  
  public:
      CSkippableRemoteResultAdaptor(IRoxieAgentContext *_ctx, IRoxieServerErrorHandler *_errorHandler, const RemoteActivityId &_remoteId, IOutputMetaData *_meta, IHThorArg &_helper, IRoxieServerActivity &_activity, bool _preserveOrder, bool _flowControlled, bool _skipping) :
          CRemoteResultAdaptor(_ctx, _errorHandler, _remoteId, _meta, _helper, _activity, _preserveOrder, _flowControlled)
      {
          skipping = _skipping;
          index = 0;
          pulled = false;
      }
  
      void setException(IException *E)
      {
          exception.setown(E);
      }
  
      virtual void onReset()
      {
          roxiemem::ReleaseRoxieRowRange(buff.getArray(), index, buff.ordinality());
          buff.kill();
          index = 0;
          pulled = false;
          exception.clear();
          CRemoteResultAdaptor::onReset();
      }
  
      void onStart(unsigned _parentExtractSize, const byte * _parentExtract)
      {
          index = 0;
          pulled = false;
          CRemoteResultAdaptor::onStart(_parentExtractSize, _parentExtract);
      }
  
      virtual const void * nextRowGE(const void *seek, const void *rawSeek, unsigned numFields, unsigned seeklen, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          // MORE - not sure what we need to do about the skip case... but we need at least this to prevent issues with exception getting lost
          if (exception)
              throw exception.getClear();
          return CRemoteResultAdaptor::nextRowGE(seek, rawSeek, numFields, seeklen, wasCompleteMatch, stepExtra);
      }
  
      virtual const void *nextRow()
      {
          if (skipping)
          {
              if(!pulled)
                  pullInput();
              if(buff.isItem(index))
              {
                  const void * next = buff.item(index++);
                  if(next)
                      processed++;
                  return next;
              }
              return NULL;
          }
          else
          {
              if (exception)
                  throw exception.getClear();
              return CRemoteResultAdaptor::nextRow();
          }
      }
  }; 
  
  //=================================================================================
  
  class CRoxieServerApplyActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorApplyArg &helper;
  
  public:
      CRoxieServerApplyActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0), helper((IHThorApplyArg &) basehelper)
      {
      }
  
      virtual void onExecute() 
      {
          helper.start();
          for (;;)
          {
              const void * next = inputStream->ungroupedNextRow();
              if (!next)
                  break;
              helper.apply(next);
              ReleaseRoxieRow(next);
          }
          helper.end();
      }
  
  };
  
  class CRoxieServerApplyActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerApplyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerApplyActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerApplyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerApplyActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerNullActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerNullActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
      }
  
      virtual const void *nextRow()
      {
          return NULL;
      }
  
  };
  
  IRoxieServerActivity * createRoxieServerNullActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
  {
      return new CRoxieServerNullActivity(_ctx, _factory, _probeManager);
  }
  
  class CRoxieServerNullActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerNullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNullActivity(_ctx, this, _probeManager);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for null activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNullActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNullSinkActivity : public CRoxieServerInternalSinkActivity
  {
  public:
      CRoxieServerNullSinkActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0)
      {
      }
  
      //override execute() to ensure that start is nevr called on any input activities.
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract) override
      {
          CriticalBlock b(ecrit);
          if (!executed)
          {
              executed = true;
              stop();
          }
      }
  
      virtual void onExecute() override
      {
      }
  };
  
  IRoxieServerActivity * createRoxieServerNullSinkActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
  {
      return new CRoxieServerNullSinkActivity(_ctx, _factory, _probeManager);
  }
  
  //=================================================================================
  
  class CRoxieServerPassThroughActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerPassThroughActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          inputStream->resetEOF();
      }
  
      virtual const void *nextRow()
      {
          const void * next = inputStream->nextRow();
          if (next)
              processed++;
          return next;
      }
  
      virtual bool isPassThrough()
      {
          return true;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (next)
              processed++;
          return next;
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  };
  
  IRoxieServerActivity * createRoxieServerPassThroughActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
  {
      return new CRoxieServerPassThroughActivity(_ctx, _factory, _probeManager);
  }
  
  //=================================================================================
  
  class CRoxieServerChildBaseActivity : public CRoxieServerActivity
  {
  protected:
      bool eof;
      bool first;
  
  public:
      CRoxieServerChildBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          eof = false;
          first = true;
      }
  
      ~CRoxieServerChildBaseActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          first = true;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  };
  
  class CRoxieServerChildBaseActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerChildBaseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const StatisticsMapping &_factoryStats)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _factoryStats)
      {
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for %s activity", getActivityText(kind));
      }
  };
  
  
  //=================================================================================
  
  class CRoxieServerChildIteratorActivity : public CRoxieServerChildBaseActivity
  {
      IHThorChildIteratorArg &helper;
  
  public:
      CRoxieServerChildIteratorActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerChildBaseActivity(_ctx, _factory, _probeManager), helper((IHThorChildIteratorArg &) basehelper)
      {
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          bool ok;
          if (first)
          {
              ok = helper.first();
              first = false;
          }
          else
              ok = helper.next();
  
          try
          {
              while (ok)
              {
                  processed++;
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  unsigned outSize = helper.transform(rowBuilder);
                  if (outSize)
                      return rowBuilder.finalizeRowClear(outSize);
                  ok = helper.next();
              }
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
  
          eof = true;
          return NULL;
      }
  
  };
  
  class CRoxieServerChildIteratorActivityFactory : public CRoxieServerChildBaseActivityFactory
  {
  public:
      CRoxieServerChildIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerChildBaseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChildIteratorActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerChildIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChildIteratorActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChildNormalizeActivity : public CRoxieServerChildBaseActivity
  {
      IHThorChildNormalizeArg &helper;
  
  public:
      CRoxieServerChildNormalizeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerChildBaseActivity(_ctx, _factory, _probeManager), helper((IHThorChildNormalizeArg &) basehelper)
      {
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          bool ok;
          if (first)
          {
              ok = helper.first();
              first = false;
          }
          else
              ok = helper.next();
  
          if (ok)
          {
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  do {
                      unsigned outSize = helper.transform(rowBuilder);
                      if (outSize)
                      {
                          processed++;
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                      ok = helper.next();
                  }
                  while (ok);
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
  
          eof = true;
          return NULL;
      }
  
  };
  
  class CRoxieServerChildNormalizeActivityFactory : public CRoxieServerChildBaseActivityFactory
  {
  public:
      CRoxieServerChildNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerChildBaseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChildNormalizeActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNewChildNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChildNormalizeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChildAggregateActivity : public CRoxieServerChildBaseActivity
  {
      IHThorChildAggregateArg &helper;
  
  public:
      CRoxieServerChildAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerChildBaseActivity(_ctx, _factory, _probeManager), helper((IHThorChildAggregateArg &) basehelper)
      {
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          eof = true;
          processed++;
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          helper.clearAggregate(rowBuilder);
          helper.processRows(rowBuilder);
          size32_t finalSize = meta.getRecordSize(rowBuilder.getSelf());
          return rowBuilder.finalizeRowClear(finalSize);
      }
  
  };
  
  class CRoxieServerChildAggregateActivityFactory : public CRoxieServerChildBaseActivityFactory
  {
  public:
      CRoxieServerChildAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerChildBaseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChildAggregateActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNewChildAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChildAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChildGroupAggregateActivity : public CRoxieServerChildBaseActivity, public IHThorGroupAggregateCallback
  {
      IHThorChildGroupAggregateArg &helper;
      RowAggregator aggregated;
  
  public:
      IMPLEMENT_IINTERFACE_USING(CRoxieServerChildBaseActivity)
      CRoxieServerChildGroupAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerChildBaseActivity(_ctx, _factory, _probeManager), helper((IHThorChildGroupAggregateArg &) basehelper),
            aggregated(helper, helper)
      {
      }
  
      void processRow(const void * next)
      {
          aggregated.addRow(next);
      }
              
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerChildBaseActivity::start(parentExtractSize, parentExtract, paused);
          aggregated.start(rowAllocator, ctx->queryCodeContext(), activityId);
      }
  
      virtual void reset()
      {
          aggregated.reset();
          CRoxieServerChildBaseActivity::reset();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          if (first)
          {
              helper.processRows(this);
              first = false;
          }
  
          Owned<AggregateRowBuilder> next = aggregated.nextResult();
          if (next)
          {
              processed++;
              return next->finalizeRowClear();
          }
          eof = true;
          return NULL;
      }
  };
  
  class CRoxieServerChildGroupAggregateActivityFactory : public CRoxieServerChildBaseActivityFactory
  {
  public:
      CRoxieServerChildGroupAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerChildBaseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChildGroupAggregateActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNewChildGroupAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChildGroupAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChildThroughNormalizeActivity : public CRoxieServerChildBaseActivity
  {
      IHThorChildThroughNormalizeArg &helper;
      const void * lastInput;
      unsigned numProcessedLastGroup;
      bool ok;
  
  public:
      CRoxieServerChildThroughNormalizeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerChildBaseActivity(_ctx, _factory, _probeManager), helper((IHThorChildThroughNormalizeArg &) basehelper)
      {
          lastInput = NULL;
          numProcessedLastGroup = 0;
          ok = false;
      }
  
      virtual void stop()
      {
          CRoxieServerChildBaseActivity::stop();
          ReleaseRoxieRow(lastInput);
          lastInput = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerChildBaseActivity::start(parentExtractSize, parentExtract, paused);
          numProcessedLastGroup = processed;
          ok = false;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          for (;;)
          {
              if (ok)
                  ok = helper.next();
  
              while (!ok)
              {
                  ReleaseRoxieRow(lastInput);
                  lastInput = inputStream->nextRow();
                  if (!lastInput)
                  {
                      if (numProcessedLastGroup != processed)
                      {
                          numProcessedLastGroup = processed;
                          return NULL;
                      }
                      lastInput = inputStream->nextRow();
                      if (!lastInput)
                          return NULL;
                  }
  
                  ok = helper.first(lastInput);
              }
  
              try
              {
                  do 
                  {
                      unsigned outSize = helper.transform(rowBuilder);
                      if (outSize)
                      {
                          processed++;
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                      ok = helper.next();
                  } while (ok);
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerChildThroughNormalizeActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerChildThroughNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChildThroughNormalizeActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNewChildThroughNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChildThroughNormalizeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerDistributionActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorDistributionArg &helper;
  
  public:
      CRoxieServerDistributionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0), helper((IHThorDistributionArg &)basehelper)
      {
      }
  
      virtual void onExecute()
      {
          MemoryAttr ma;
          IDistributionTable * * accumulator = (IDistributionTable * *)ma.allocate(helper.queryInternalRecordSize()->getMinRecordSize());
          helper.clearAggregate(accumulator);
  
          OwnedConstRoxieRow nextrec(inputStream->nextRow());
          for (;;)
          {
              if (!nextrec)
              {
                  nextrec.setown(inputStream->nextRow());
                  if (!nextrec)
                      break;
              }
              helper.process(accumulator, nextrec);
              nextrec.setown(inputStream->nextRow());
          }
          StringBuffer result;
          result.append("<XML>");
          helper.gatherResult(accumulator, result);
          result.append("</XML>");
          helper.sendResult(result.length(), result.str());
          helper.destruct(accumulator);
      }
  };
  
  class CRoxieServerDistributionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerDistributionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerDistributionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDistributionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerDistributionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  
  
  //=================================================================================
  
  class CRoxieServerLinkedRawIteratorActivity : public CRoxieServerActivity
  {
      IHThorLinkedRawIteratorArg &helper;
  
  public:
      CRoxieServerLinkedRawIteratorActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorLinkedRawIteratorArg &) basehelper)
      {
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void *ret =helper.next();
          if (ret)
          {
              LinkRoxieRow(ret);
              processed++;
          }
          return ret;
      }
  
  };
  
  class CRoxieServerLinkedRawIteratorActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerLinkedRawIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLinkedRawIteratorActivity(_ctx, this, _probeManager);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for %s activity", getActivityText(kind));
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLinkedRawIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerLinkedRawIteratorActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerDatasetResultActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerDatasetResultActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected();
      }
  
      virtual void executeChild(size32_t & retSize, void * & ret, unsigned parentExtractSize, const byte * parentExtract)
      {
          try
          {
              start(parentExtractSize, parentExtract, false);
              {
                  ActivityTimer t(activityStats, timeActivities);
                  MemoryBuffer result;
                  IRecordSize * inputMeta = input->queryOutputMeta();
                  for (;;)
                  {
                      const void *nextrec = inputStream->ungroupedNextRow();
                      if (!nextrec)
                          break;
                      result.append(inputMeta->getRecordSize(nextrec), nextrec);
                      ReleaseRoxieRow(nextrec);
                  }
                  retSize = result.length();
                  ret = result.detach();
              }
              stop();
              reset(); 
          }
          catch(IException *E)
          {
              ctx->notifyAbort(E);
              abort();
              reset(); 
              throw;
          }
          catch(...)
          {
              Owned<IException> E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught at %s:%d", sanitizeSourceFile(__FILE__), __LINE__);
              ctx->notifyAbort(E);
              abort();
              reset(); 
              throw;
          }
      }
  
  
  };
  
  class CRoxieServerDatasetResultActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerDatasetResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerDatasetResultActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerDatasetResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerDatasetResultActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerInlineTableActivity : public CRoxieServerActivity
  {
      IHThorInlineTableArg &helper;
      __uint64 curRow;
      __uint64 numRows;
  
  public:
      CRoxieServerInlineTableActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorInlineTableArg &) basehelper)
      {
          curRow = 0;
          numRows = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          curRow = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numRows = helper.numRows();
      }
  
      virtual bool needsAllocator() const { return true; }
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          // Filtering empty rows, returns the next valid row
          while (curRow < numRows)
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned outSize = helper.getRow(rowBuilder, curRow++);
              if (outSize)
              {
                  processed++;
                  return rowBuilder.finalizeRowClear(outSize);
              }
          }
          return NULL;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  };
  
  class CRoxieServerStrandedInlineTableActivity : public CRoxieServerStrandedActivity
  {
      IHThorInlineTableArg &helper;
      __uint64 numRows;
  
      //An inline table that isn't stranded
      class InlineTableSimpleProcessor : public StrandProcessor
      {
      protected:
          IHThorInlineTableArg &helper;
          __uint64 curRow;
          __uint64 maxRows;
  
      public:
          InlineTableSimpleProcessor(CRoxieServerActivity &_parent, IHThorInlineTableArg &_helper)
              : StrandProcessor(_parent, NULL, true), helper(_helper), curRow(0), maxRows(0)
          {
          }
          virtual void start() override
          {
              StrandProcessor::start();
              curRow = 0;
              maxRows = queryParent().numRows;
          }
          virtual const void * nextRow()
          {
              ActivityTimer t(activityStats, timeActivities);
              //Return rows while the current section is active
              while (curRow < maxRows)
              {
                  RtlDynamicRowBuilder rowBuilder(*rowAllocator);
                  unsigned outSize = helper.getRow(rowBuilder, curRow++);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
  
              return NULL;
          }
          CRoxieServerStrandedInlineTableActivity & queryParent() { return static_cast<CRoxieServerStrandedInlineTableActivity&>(parent); }
      };
  
      class InlineTableStrandProcessor : public StrandProcessor
      {
      protected:
          IHThorInlineTableArg &helper;
          unsigned whichStrand;
          unsigned numStrands = 0;
          __uint64 sectionSize;
          __uint64 curRow = 0;
          __uint64 sectionMaxRows = 0;
          __uint64 maxRows = 0;
          bool isOrdered;
          bool eosPending;
  
      public:
          InlineTableStrandProcessor(CRoxieServerActivity &_parent, IHThorInlineTableArg &_helper, unsigned _whichStrand, bool _isOrdered)
              : StrandProcessor(_parent, NULL, true), helper(_helper), whichStrand(_whichStrand), isOrdered(_isOrdered)
          {
              sectionSize = queryParent().strandOptions.blockSize;
              eosPending = false;
          }
          virtual void start() override
          {
              StrandProcessor::start();
              numStrands = queryParent().numStrands();
              maxRows = queryParent().numRows;
              curRow = std::min(whichStrand * sectionSize, maxRows);
              sectionMaxRows = std::min(curRow + sectionSize, maxRows);
              eosPending = isOrdered && (curRow != maxRows);
          }
          virtual const void * nextRow()
          {
              ActivityTimer t(activityStats, timeActivities);
              for (;;)
              {
                  //Return rows while the current section is active
                  while (curRow < sectionMaxRows)
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      unsigned outSize = helper.getRow(rowBuilder, curRow++);
                      if (outSize)
                      {
                          processed++;
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                  }
  
                  if (curRow == maxRows)
                      return NULL;
  
                  //Update current position to the next section
                  curRow += (numStrands - 1) * sectionSize;
                  if (curRow > maxRows)
                      curRow = maxRows;
                  sectionMaxRows = std::min(curRow + sectionSize, maxRows);
  
                  //If we have output a block of rows, then send the block as a unit - results now come from the next strand
                  if (eosPending)
                  {
                      //If there is another block, then second an end of section marker after it, otherwise make sure NULL is returned next.
                      eosPending = isOrdered && (curRow != maxRows);
                      return queryEndOfSectionMarker();
                  }
              }
          }
          CRoxieServerStrandedInlineTableActivity & queryParent() { return static_cast<CRoxieServerStrandedInlineTableActivity&>(parent); }
      };
  
  public:
      CRoxieServerStrandedInlineTableActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const StrandOptions &_strandOptions)
          : CRoxieServerStrandedActivity(_ctx, _factory, _probeManager, _strandOptions), helper((IHThorInlineTableArg &) basehelper)
      {
          numRows = 0;
      }
  
      virtual StrandProcessor *createStrandProcessor(IEngineRowStream *instream)  { throwUnexpected(); }
      virtual StrandProcessor *createStrandSourceProcessor(bool inputOrdered)
      {
          if (strandOptions.numStrands <= 1)
              return new InlineTableSimpleProcessor(*this, helper);
          return new InlineTableStrandProcessor(*this, helper, strands.ordinality(), inputOrdered);
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerStrandedActivity::start(parentExtractSize, parentExtract, paused);
          numRows = helper.numRows();
          onStartStrands();
          startJunction(sourceJunction); // This must be started *after* all the strands have been initialised
      }
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  };
  
  class CRoxieServerInlineTableActivityFactory : public CRoxieServerActivityFactory
  {
      StrandOptions strandOptions;
  
  public:
      CRoxieServerInlineTableActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), strandOptions(_graphNode)
      {
          optStableInput = false; // do not force the output to be ordered
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerStrandedInlineTableActivity(_ctx, this, _probeManager, strandOptions);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for InlineTable activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerInlineTableActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerInlineTableActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerWorkUnitReadActivity : public CRoxieServerActivity
  {
      IHThorWorkunitReadArg &helper;
      CriticalSection readerCrit;
      Owned<IWorkUnitRowReader> wuReader;
  public:
      CRoxieServerWorkUnitReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorWorkunitReadArg &)basehelper)
      {
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          if (!ctx->queryServerContext())
          {
              throw MakeStringException(ROXIE_INTERNAL_ERROR, "Workunit read activity cannot be executed in agent context");
          }
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          IXmlToRowTransformer * xmlTransformer = helper.queryXmlTransformer();
          OwnedRoxieString fromWuid(helper.getWUID());
          wuReader.setown(ctx->getWorkunitRowReader(fromWuid, helper.queryName(), helper.querySequence(), xmlTransformer, rowAllocator, meta.isGrouped()));
          // MORE _ should that be in onCreate?
      }
  
      virtual void reset() 
      {
          {
              CriticalBlock b(readerCrit);
              wuReader.clear();
          }
          CRoxieServerActivity::reset(); 
      };
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          Linked<IWorkUnitRowReader> useReader;
          {
              CriticalBlock b(readerCrit);
              if (!wuReader)
                  return NULL;
              useReader.set(wuReader);
          }
          const void *ret = useReader->nextRow();
          if (ret)
              processed++;
          return ret;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  };
  
  class CRoxieServerWorkUnitReadActivityFactory : public CRoxieServerActivityFactory
  {
  
  public:
      CRoxieServerWorkUnitReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerWorkUnitReadActivity(_ctx, this, _probeManager);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for WorkUnitRead activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerWorkUnitReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerWorkUnitReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  interface ILocalGraphEx : public IEclGraphResults
  {
  public:
      virtual void setResult(unsigned id, IGraphResult * result) = 0;
      virtual IEngineRowStream * createResultIterator(unsigned id) = 0;
      virtual void setGraphLoopResult(IGraphResult * result) = 0;
      virtual IEngineRowStream * createGraphLoopResultIterator(unsigned id) = 0;
  };
  
  
  class CSafeRoxieInput : implements IEngineRowStream, implements IFinalRoxieInput, public CInterface
  {
  public:
      CSafeRoxieInput(unsigned _sourceIdx, IFinalRoxieInput * _input) : input(_input), inputStream(NULL), sourceIdx(_sourceIdx) {}
      IMPLEMENT_IINTERFACE
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          connectInputStreams(ctx, consumerOrdered);
          streams.append(this);
          return NULL;
      }
      void connectInputStreams(IRoxieAgentContext *ctx, bool consumerOrdered)
      {
          inputStream.set(connectSingleStream(ctx, input, sourceIdx, junction, consumerOrdered));
      }
      virtual IOutputMetaData * queryOutputMeta() const
      {
          return input->queryOutputMeta();
      }
      virtual unsigned __int64 queryTotalCycles() const
      {
          return input->queryTotalCycles();
      }
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
      virtual IRoxieServerActivity *queryActivity()
      {
          return input->queryActivity();
      }
      virtual IIndexReadActivityInfo *queryIndexReadActivity()
      {
          return input->queryIndexReadActivity();
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CriticalBlock procedure(cs);
          input->start(parentExtractSize, parentExtract, paused);
          startJunction(junction);
      }
      virtual void stop()
      {
          CriticalBlock procedure(cs);
          inputStream->stop();
      }
      virtual void reset()
      {
          CriticalBlock procedure(cs);
          input->reset();
          resetJunction(junction);
      }
      virtual void resetEOF()
      {
          CriticalBlock procedure(cs);
          inputStream->resetEOF();
      }
      virtual const void *nextRow()
      {
          CriticalBlock procedure(cs);
          return inputStream->nextRow();
      }
      virtual bool nextGroup(ConstPointerArray & group)
      {
          CriticalBlock procedure(cs);
          return inputStream->nextGroup(group);
      }
  
  private:
      CriticalSection cs;
      Linked<IFinalRoxieInput> input;
      Linked<IEngineRowStream> inputStream;
      Owned<IStrandJunction> junction;
      unsigned sourceIdx;
  };
  
  
  //=================================================================================
  
  class CPseudoRoxieInput : implements IEngineRowStream, implements IFinalRoxieInput, public CInterface
  {
  public:
      IMPLEMENT_IINTERFACE;
      
      CPseudoRoxieInput()
      { 
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return 0;
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
  
      virtual IRoxieServerActivity *queryActivity()
      {
          throwUnexpected();
      }
      virtual IIndexReadActivityInfo *queryIndexReadActivity()
      {
          throwUnexpected();
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          assertex(!idx);
          streams.append(this);
          return NULL;
      }
  
      virtual IOutputMetaData * queryOutputMeta() const { throwUnexpected(); }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) { }
      virtual void stop() { }
      virtual void reset() { }
      virtual void checkAbort() { }
      virtual void resetEOF() { }
  };
  
  
  class CIndirectRoxieInput : public CPseudoRoxieInput
  {
  public:
      CIndirectRoxieInput() : input(NULL)
      {
          stream = NULL;
          sourceIdx = 0;
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) 
      {
          input->start(parentExtractSize, parentExtract, paused);
          startJunction(junction);
      }
      virtual void stop()
      {
          stream->stop();
      }
      virtual void reset()
      { 
          input->reset();
          resetJunction(junction);
      }
  
      virtual const void * nextRow()
      {
          return stream->nextRow();
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          return stream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return input ? input->queryTotalCycles() : 0;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
      { 
          return input->gatherConjunctions(collector);
      }
  
      virtual void resetEOF() 
      { 
          stream->resetEOF();
      }
  
      virtual unsigned numConcreteOutputs() const 
      {
          return input->numConcreteOutputs();
      }
      
      virtual IFinalRoxieInput * queryConcreteInput(unsigned idx)
      { 
          return input->queryConcreteInput(idx);
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
      {
          return input->queryConcreteOutputStream(whichInput);
      }
  
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
      {
          return input->queryConcreteOutputJunction(idx);
      }
  
      virtual IOutputMetaData * queryOutputMeta() const 
      { 
          return input->queryOutputMeta(); 
      }
  
      virtual IRoxieServerActivity *queryActivity() 
      {
          return input->queryActivity();
      }
  
      inline void setInput(unsigned _sourceIdx, IFinalRoxieInput * _input)
      {
          input = _input;
          sourceIdx = _sourceIdx;
      }
      inline void connectInputStreams(IRoxieAgentContext *ctx, bool consumerOrdered)
      {
          stream = connectSingleStream(ctx, input, sourceIdx, junction, consumerOrdered);
      }
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          assertex(idx == 0);
          connectInputStreams(ctx, consumerOrdered);
          streams.append(this);
          return NULL;
      }
      inline IFinalRoxieInput *queryInput() const
      {
          return input;
      }
  protected:
      unsigned sourceIdx;
      IFinalRoxieInput *input;
      IEngineRowStream *stream;
      Owned<IStrandJunction> junction;
  };
  
  
  class CExtractMapperInput : public CIndirectRoxieInput
  {
      unsigned savedParentExtractSize;
      const byte * savedParentExtract;
  public:
      CExtractMapperInput()
      {
          savedParentExtractSize = 0;
          savedParentExtract = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CIndirectRoxieInput::start(savedParentExtractSize, savedParentExtract, paused);
      }
  
      void setParentExtract(unsigned _savedParentExtractSize, const byte * _savedParentExtract)
      {
          savedParentExtractSize = _savedParentExtractSize;
          savedParentExtract = _savedParentExtract;
      }
  };
  
  
  class CGraphResult : implements IGraphResult, public CInterface
  {
      CriticalSection cs;
      const byte **rowset;
      size32_t count;
      bool complete;
  
      void clear()
      {
          CriticalBlock func(cs);
          rtlReleaseRowset(count, rowset);
          rowset = NULL;
          count = 0;
          complete = false;
      }
  
  public:
      IMPLEMENT_IINTERFACE
      CGraphResult() 
      { 
          complete = false; // dummy result is not supposed to be used...
          rowset = NULL;
          count = 0;
      }
  
      CGraphResult(size32_t _count, const byte **_rowset)
      : rowset(_rowset), count(_count)
      { 
          complete = true;
      }
      ~CGraphResult()
      {
          clear();
      }
  
  // interface IGraphResult
      virtual IEngineRowStream * createIterator()
      {
          if (!complete)
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Internal Error: Reading uninitialised graph result"); 
          return new CGraphResultIterator(this);
      }
  
      virtual void getLinkedResult(unsigned & countResult, const byte * * & result) override
      {
          if (!complete)
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Internal Error: Reading uninitialised graph result"); 
  
          result = rtlLinkRowset(rowset);
          countResult = count;
       }
  
      virtual const void * getLinkedRowResult()
      {
          if (!complete)
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Internal Error: Reading uninitialised graph result");
  
          if (count != 1)
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Internal Error: Expected a single row result");
          const void * ret = rowset[0];
          LinkRoxieRow(ret);
          return ret;
      }
  
  //other 
      const void * getRow(unsigned i)
      {
          CriticalBlock func(cs);
          if (i >= count)
              return NULL;
          const void * ret = rowset[i];
          if (ret) LinkRoxieRow(ret);
          return ret;
      }
  
  protected:
      class CGraphResultIterator : public CInterfaceOf<IEngineRowStream>
      {
          unsigned i;
          Linked<CGraphResult> result;
  
      public:
          CGraphResultIterator(CGraphResult * _result) : result(_result) { i = 0; }
  
      public:
          virtual const void * nextRow()
          {
              return result->getRow(i++);
          }
          virtual void stop()
          {
          }
          virtual void resetEOF()
          {
              throwUnexpected();
          }
  
      };
  };
  
  
  //=================================================================================
  
  class CRoxieServerLocalResultReadActivity : public CRoxieServerActivity
  {
      IHThorLocalResultReadArg &helper;
      CriticalSection iterCrit;
      Owned<IEngineRowStream> iter;
      ILocalGraphEx * graph;
      unsigned graphId;
      unsigned sequence;
  
  public:
      CRoxieServerLocalResultReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorLocalResultReadArg &)basehelper), graphId(_graphId)
      {
          graph = NULL;
          sequence = 0;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          graph = static_cast<ILocalGraphEx *>(ctx->queryCodeContext()->resolveLocalQuery(graphId));
          sequence = helper.querySequence();
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          iter.setown(graph->createResultIterator(sequence));
      }
  
      virtual void reset() 
      {
          {
              CriticalBlock b(iterCrit);
              iter.clear();
          }
          CRoxieServerActivity::reset(); 
      };
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          Linked<IEngineRowStream> useIter;
          {
              CriticalBlock b(iterCrit);
              if (!iter)
                  return NULL;
              useIter.set(iter);
          }
          const void * next = useIter->nextRow();
          if (next)
          {
              processed++;
              rowsIn++;
          }
          return next;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  };
  
  class CRoxieServerLocalResultReadActivityFactory : public CRoxieServerActivityFactory
  {
       unsigned graphId;
  
  public:
      CRoxieServerLocalResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _graphId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), graphId(_graphId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLocalResultReadActivity(_ctx, this, _probeManager, graphId);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for LocalResultRead activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLocalResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned graphId)
  {
      return new CRoxieServerLocalResultReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, graphId);
  }
  
  //=================================================================================
  
  class CRoxieServerLocalResultStreamReadActivity : public CRoxieServerActivity
  {
      IHThorLocalResultReadArg &helper;
      unsigned sequence;
  
  public:
      CRoxieServerLocalResultStreamReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorLocalResultReadArg &)basehelper)
      {
          sequence = 0;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          sequence = helper.querySequence();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = inputStream->nextRow();
          if (next)
          {
              processed++;
              rowsIn++;
          }
          return next;
      }
  
      virtual bool querySetStreamInput(unsigned id, unsigned _sourceIdx, IFinalRoxieInput * _input)
      {
          if (id == sequence)
          {
              CRoxieServerActivity::setInput(0, _sourceIdx, _input);
              connectInputStreams(true);
              return true;
          }
          return false;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  };
  
  class CRoxieServerLocalResultStreamReadActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerLocalResultStreamReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLocalResultStreamReadActivity(_ctx, this, _probeManager);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for LocalResultRead activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLocalResultStreamReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerLocalResultStreamReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerLocalResultWriteActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorLocalResultWriteArg &helper;
      ILocalGraphEx * graph;
      unsigned graphId;
  
  public:
      CRoxieServerLocalResultWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorLocalResultWriteArg &)basehelper), graphId(_graphId)
      {
          graph = NULL;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
          graph = static_cast<ILocalGraphEx *>(ctx->queryCodeContext()->resolveLocalQuery(graphId));
      }
  
      virtual void onExecute() 
      {
          RtlLinkedDatasetBuilder builder(rowAllocator);
          inputStream->readAll(builder);
          Owned<CGraphResult> result = new CGraphResult(builder.getcount(), builder.linkrows());
          graph->setResult(helper.querySequence(), result);
      }
  
      virtual const void *nextRow()
      {
          return inputStream->nextRow(); // I can act as a passthrough input
      }
  
      IFinalRoxieInput * querySelectOutput(unsigned id)
      {
          if (id == helper.querySequence())
          {
              executed = true;  // Ensure that we don't try to pull as a sink as well as via the passthrough
              return LINK(this);
          }
          return NULL;
      }
  };
  
  class CRoxieServerLocalResultWriteActivityFactory : public CRoxieServerInternalSinkFactory
  {
      unsigned graphId;
  public:
      CRoxieServerLocalResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot), graphId(_graphId)
      {
          isInternal = true;
          Owned<IHThorLocalResultWriteArg> helper = (IHThorLocalResultWriteArg *) helperFactory();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLocalResultWriteActivity(_ctx, this, _probeManager, graphId, usageCount);
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerLocalResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId, bool _isRoot)
  {
      return new CRoxieServerLocalResultWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _graphId, _isRoot);
  }
  
  //=====================================================================================================
  
  class CRoxieServerDictionaryResultWriteActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorDictionaryResultWriteArg &helper;
      ILocalGraphEx * graph;
      unsigned graphId;
  
  public:
      CRoxieServerDictionaryResultWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _usageCount, unsigned _graphId)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _usageCount), helper((IHThorDictionaryResultWriteArg &)basehelper), graphId(_graphId)
      {
          graph = NULL;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
          graph = static_cast<ILocalGraphEx *>(ctx->queryCodeContext()->resolveLocalQuery(graphId));
      }
  
      virtual void onExecute()
      {
          RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
          for (;;)
          {
              const void *row = inputStream->ungroupedNextRow();
              if (!row)
                  break;
              builder.appendOwn(row);
              processed++;
          }
          Owned<CGraphResult> result = new CGraphResult(builder.getcount(), builder.linkrows());
          graph->setResult(helper.querySequence(), result);
      }
  
      virtual const void *nextRow()
      {
          return inputStream->nextRow(); // I can act as a passthrough input
      }
  
      IFinalRoxieInput * querySelectOutput(unsigned id)
      {
          if (id == helper.querySequence())
          {
              executed = true;  // Ensure that we don't try to pull as a sink as well as via the passthrough
              return LINK(this);
          }
          return NULL;
      }
  };
  
  class CRoxieServerDictionaryResultWriteActivityFactory : public CRoxieServerInternalSinkFactory
  {
      unsigned graphId;
  public:
      CRoxieServerDictionaryResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot), graphId(_graphId)
      {
          isInternal = true;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerDictionaryResultWriteActivity(_ctx, this, _probeManager, usageCount, graphId);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDictionaryResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId, bool _isRoot)
  {
      return new CRoxieServerDictionaryResultWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _graphId, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerGraphLoopResultReadActivity : public CRoxieServerActivity
  {
  protected:
      IHThorGraphLoopResultReadArg &helper;
      CriticalSection iterCrit;
      Owned<IFinalRoxieInput> iterInput;
      Owned<IEngineRowStream> iterStream;
      Owned<IStrandJunction> iterJunction;
      ILocalGraphEx * graph;
      unsigned graphId;
      unsigned sequence;
  
  public:
      CRoxieServerGraphLoopResultReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorGraphLoopResultReadArg &)basehelper), graphId(_graphId)
      {
          graph = NULL;
          sequence = 0;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          graph = static_cast<ILocalGraphEx *>(ctx->queryCodeContext()->resolveLocalQuery(graphId));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (iterInput)
              iterInput->start(parentExtractSize, parentExtract, paused);
          else
          {
              sequence = helper.querySequence();
              if ((int)sequence >= 0)
              {
                  try
                  {
                      iterStream.setown(graph->createGraphLoopResultIterator(sequence));
                      iterInput.clear();
                  }
                  catch (IException * E)
                  {
                      throw makeWrappedException(E);
                  }
              }
          }
          startJunction(iterJunction);
      }
  
      virtual void stop()
      {
          if (iterStream)
              iterStream->stop();
          CRoxieServerActivity::stop();
      }
  
      virtual void reset() 
      {
          {
              CriticalBlock b(iterCrit);
              if (iterInput)
                  iterInput->reset();
              resetJunction(iterJunction);
              iterInput.clear();
              iterStream.clear();
              iterJunction.clear();
          }
          CRoxieServerActivity::reset(); 
      };
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          Linked<IEngineRowStream> useStream;
          {
              CriticalBlock b(iterCrit);
              if (!iterStream)
                  return NULL;
              useStream.set(iterStream);
          }
          const void * next = useStream->nextRow();
          if (next)
          {
              processed++;
              rowsIn++;
          }
          return next;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  
      virtual void gatherIterationUsage(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract)
      {
          ensureCreated();
          basehelper.onStart(parentExtract, NULL);
          processor.noteUseIteration(helper.querySequence());
      }
  
      virtual void associateIterationOutputs(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract, IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes)
      {
          //helper already initialised from the gatherIterationUsage() call.
          CRoxieServerActivity::connectInputStreams(true);
          iterInput.set(processor.connectIterationOutput(helper.querySequence(), probeManager, probes, this, 0));
          iterStream.set(connectSingleStream(ctx, iterInput, 0, iterJunction, true));
      }
  
      virtual IInputSteppingMeta * querySteppingMeta()
      {
          assertex(iterInput);
          return iterInput->querySteppingMeta();
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          assertex(iterStream);
          return iterStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
      }
  };
  
  
  //variety of CRoxieServerGraphLoopResultReadActivity created internally with a predefined sequence number
  class CRoxieServerInternalGraphLoopResultReadActivity : public CRoxieServerGraphLoopResultReadActivity
  {
  public:
      CRoxieServerInternalGraphLoopResultReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId, unsigned _sequence)
          : CRoxieServerGraphLoopResultReadActivity(_ctx, _factory, _probeManager, _graphId)
      {
          sequence = _sequence;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if ((int)sequence >= 0)
          {
              try
              {
                  iterStream.setown(graph->createGraphLoopResultIterator(sequence));
                  iterInput.clear();
              }
              catch (IException * E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  
  class CRoxieServerGraphLoopResultReadActivityFactory : public CRoxieServerActivityFactory
  {
       unsigned graphId;
  
  public:
      CRoxieServerGraphLoopResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _graphId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), graphId(_graphId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerGraphLoopResultReadActivity(_ctx, this, _probeManager, graphId);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for GraphLoopResultRead activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerGraphLoopResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned graphId)
  {
      return new CRoxieServerGraphLoopResultReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, graphId);
  }
  
  //=====================================================================================================
  
  class CRoxieServerGraphLoopResultWriteActivity : public CRoxieServerInternalSinkActivity
  {
      ILocalGraphEx * graph;
      unsigned graphId;
  
  public:
      CRoxieServerGraphLoopResultWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), graphId(_graphId)
      {
          graph = NULL;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
          graph = static_cast<ILocalGraphEx *>(ctx->queryCodeContext()->resolveLocalQuery(graphId));
      }
  
      virtual void onExecute() 
      {
          RtlLinkedDatasetBuilder builder(rowAllocator);
          inputStream->readAll(builder);
          Owned<CGraphResult> result = new CGraphResult(builder.getcount(), builder.linkrows());
          graph->setGraphLoopResult(result);
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==0)
              return this;
          else
              return NULL;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          inputStream->resetEOF();
      }
  
      virtual const void *nextRow()
      {
          const void * next = inputStream->nextRow();
          if (next)
              processed++;
          return next;
      }
  
      virtual bool isPassThrough()
      {
          return true;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (next)
              processed++;
          return next;
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
      virtual IOutputMetaData * queryOutputMeta() const 
      { 
          return input->queryOutputMeta(); 
      }
  };
  
  class CRoxieServerGraphLoopResultWriteActivityFactory : public CRoxieServerInternalSinkFactory
  {
      unsigned graphId;
  public:
      CRoxieServerGraphLoopResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, true), graphId(_graphId)
      {
          isInternal = true;
          Owned<IHThorGraphLoopResultWriteArg> helper = (IHThorGraphLoopResultWriteArg *) helperFactory();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerGraphLoopResultWriteActivity(_ctx, this, _probeManager, graphId, usageCount);
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerGraphLoopResultWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, unsigned _graphId)
  {
      return new CRoxieServerGraphLoopResultWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _graphId);
  }
  
  //=================================================================================
  
  class CRoxieServerDedupActivity : public CRoxieServerActivity
  {
  protected:
      IHThorDedupArg &helper;
      unsigned numKept;
      unsigned numToKeep;
  
  public:
  
      CRoxieServerDedupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorDedupArg &)basehelper)
      {
          numKept = 0;
          numToKeep = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numKept = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numToKeep = helper.numToKeep();
      }
  };
  
  class CRoxieServerDedupKeepLeftActivity : public CRoxieServerDedupActivity
  {
      IRangeCompare * stepCompare;
      const void *prev;
  
  public:
  
      CRoxieServerDedupKeepLeftActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerDedupActivity(_ctx, _factory, _probeManager)
      {
          prev = NULL;
          stepCompare = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          prev = NULL;
          CRoxieServerDedupActivity::start(parentExtractSize, parentExtract, paused);
          IInputSteppingMeta * stepMeta = input->querySteppingMeta();
          stepCompare = NULL;
          if (stepMeta)
              stepCompare = stepMeta->queryCompare();
      }
  
      virtual void reset()
      {
          ReleaseClearRoxieRow(prev);
          CRoxieServerDedupActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next;
          for (;;)
          {
              next = inputStream->nextRow();
              if (!prev || !next || !helper.matches(prev,next))
              {
                  numKept = 0;
                  break;
              }
  
              if (numKept < numToKeep-1)
              {
                  numKept++;
                  break;
              }
  
              ReleaseRoxieRow(next);
          }
  
          ReleaseRoxieRow(prev);
          prev = next;
          if (next)
          {
              LinkRoxieRow(next);
              processed++;
          }
          return next;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next;
          for (;;)
          {
              next = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  
              //If the record was an in-exact match from the index then return it immediately
              //and don't cause it to dedup following legal records.
              if (!wasCompleteMatch)
              {
                  assertex(stepExtra.returnMismatches());
                  return next;
              }
  
              if (!prev || !next || !helper.matches(prev,next))
              {
                  numKept = 0;
                  break;
              }
  
              if (numKept < numToKeep-1)
              {
                  numKept++;
                  break;
              }
  
              //Unusual - deduping by x,y stepped on x,y,z - still want any record back as soon as possible.
              if (stepExtra.returnMismatches())
              {
                  //If asked to return mismatches we are only interested in mismatches that will force the stepped
                  //condition to advance
                  if (stepCompare->docompare(next, seek, numFields) != 0)
                  {
                      wasCompleteMatch = false;
                      break;
                  }
              }
  
              ReleaseRoxieRow(next);
          }
  
          ReleaseRoxieRow(prev);
          prev = next;
          if (next)
          {
              LinkRoxieRow(next);
              processed++;
          }
          return next;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
      {
          return input->gatherConjunctions(collector);
      }
  
      virtual void resetEOF()
      {
          inputStream->resetEOF();
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  };
  
  //=================================================================================
  
  class CRoxieServerDedupKeepRightActivity : public CRoxieServerDedupActivity
  {
      const void *kept;
      bool first;
      ICompare * compareBest;
  
  public:
  
      CRoxieServerDedupKeepRightActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerDedupActivity(_ctx, _factory, _probeManager), compareBest(nullptr)
      {
          kept = nullptr;
          first = true;
          if (helper.keepBest())
              compareBest = helper.queryCompareBest();
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          first = true;
          kept = nullptr;
          CRoxieServerDedupActivity::start(parentExtractSize, parentExtract, paused);
          if (numToKeep>1)
              throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "DEDUP with RIGHT and NumToKeep>1 not supported");
      }
  
      virtual void reset()
      {
          ReleaseClearRoxieRow(kept);
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (first)
          {
              kept = inputStream->nextRow();
              first = false;
          }
          const void * next;
          for (;;)
          {
              next = inputStream->nextRow();
              if (!kept || !next || !helper.matches(kept,next))
                  break;
  
              if (compareBest && compareBest->docompare(kept,next) <= 0)
                  continue;
  
              ReleaseRoxieRow(kept);
              kept = next;
          }
  
          const void * ret = kept;
          kept = next;
  //      CTXLOG("dedup returns %p", ret);
          if (ret) processed++;
          return ret;
      }
  };
  
  class CRoxieServerDedupAllActivity : public CRoxieServerActivity
  {
      IHThorDedupArg &helper;
      unsigned     survivorIndex;
      ConstPointerArray survivors;
      bool keepLeft;
      bool eof;
      bool first;
      ICompare *primaryCompare;
  
  public:
      CRoxieServerDedupAllActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _keepLeft)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorDedupArg &)basehelper)
      {
          keepLeft = _keepLeft;
          primaryCompare = helper.queryComparePrimary();
          eof = false;
          first = true;
          survivorIndex = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          first = true;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          while (survivors.isItem(survivorIndex))
          {
              ReleaseRoxieRow(survivors.item(survivorIndex++));
          }
          survivors.kill();
          eof = false;
          first = true;
          CRoxieServerActivity::reset();
      }
  
      void dedupRange(unsigned first, unsigned last, ConstPointerArray & group)
      {
          for (unsigned idxL = first; idxL < last; idxL++)
          {
              const void * left = group.item(idxL);
              if (left)
              {
                  for (unsigned idxR = first; idxR < last; idxR++)
                  {
                      const void * right = group.item(idxR);
                      if ((idxL != idxR) && right)
                      {
                          if (helper.matches(left, right))
                          {
                              if (keepLeft)
                              {
                                  group.replace(NULL, idxR);
                                  ReleaseRoxieRow(right);
                              }
                              else
                              {
                                  group.replace(NULL, idxL);
                                  ReleaseRoxieRow(left);
                                  break;
                              }
                          }
                      }
                  }
              }
          }
      }
  
      void dedupRange(unsigned first, unsigned last, void ** rows)
      {
          for (unsigned idxL = first; idxL < last; idxL++)
          {
              void * left = rows[idxL];
              if (left)
              {
                  for (unsigned idxR = first; idxR < last; idxR++)
                  {
                      void * right = rows[idxR];
                      if ((idxL != idxR) && right)
                      {
                          if (helper.matches(left, right))
                          {
                              if (keepLeft)
                              {
                                  rows[idxR] = NULL;
                                  ReleaseRoxieRow(right);
                              }
                              else
                              {
                                  rows[idxL] = NULL;
                                  ReleaseRoxieRow(left);
                                  break;
                              }
                          }
                      }
                  }
              }
          }
      }
  
      bool calcNextDedupAll()
      {
          survivors.kill();
          survivorIndex = 0;
  
          ConstPointerArray group;
          if (eof || !inputStream->nextGroup(group))
          {
              eof = true;
              return false;
          }
  
          unsigned max = group.ordinality();
          if (primaryCompare)
          {
              //hard, if not impossible, to hit this code once optimisations in place
              MemoryAttr indexbuff(max*sizeof(void *));
              void ** temp = (void **)indexbuff.bufferBase();
              void * *rows = const_cast<void * *>(group.getArray());
              msortvecstableinplace(rows, max, *primaryCompare, temp);
              unsigned first = 0;
              for (unsigned idx = 1; idx < max; idx++)
              {
                  if (primaryCompare->docompare(rows[first], rows[idx]) != 0)
                  {
                      dedupRange(first, idx, rows);
                      first = idx;
                  }
              }
              dedupRange(first, max, rows);
  
              for(unsigned idx2=0; idx2<max; ++idx2)
              {
                  void * cur = rows[idx2];
                  if(cur)
                      survivors.append(cur);
              }
          }
          else
          {
              dedupRange(0, max, group);
              for(unsigned idx=0; idx<max; ++idx)
              {
                  const void * cur = group.item(idx);
                  if(cur)
                      survivors.append(cur);
              }
          }
  
          return true;
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (first)
          {
              calcNextDedupAll();
              first = false;
          }
          while (survivors.isItem(survivorIndex))
          {
              const void *ret = survivors.item(survivorIndex++);
              if (ret)
              {
                  processed++;
                  return ret;
              }
          }
          calcNextDedupAll();
          return NULL;
      }
  
  };
  
  class CRoxieServerDedupActivityFactory : public CRoxieServerActivityFactory
  {
      bool compareAll;
      bool keepLeft;
      bool keepBest;
  
  public:
      CRoxieServerDedupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorDedupArg> helper = (IHThorDedupArg *) helperFactory();
          compareAll = helper->compareAll();
          keepLeft = helper->keepLeft();
          keepBest = helper->keepBest();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (compareAll)
              return new CRoxieServerDedupAllActivity(_ctx, this, _probeManager, keepLeft);
          else if (keepLeft && !keepBest)
              return new CRoxieServerDedupKeepLeftActivity(_ctx, this, _probeManager);
          else
              return new CRoxieServerDedupKeepRightActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDedupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerDedupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerHashDedupActivity : public CRoxieServerActivity
  {
      bool eof;
      IHThorHashDedupArg &helper;
  
      class HashDedupElement
      {
      public:
          HashDedupElement(unsigned _hash, const void *_keyRow)
              : hash(_hash), keyRow(_keyRow)
          {
          }
          ~HashDedupElement()
          {
              ReleaseRoxieRow(keyRow);
          }
          inline unsigned queryHash() const
          {
              return hash;
          }
          inline const void *queryRow() const
          {
              return keyRow;
          }
          inline const void *getRowClear()
          {
              const void * row = keyRow;
              keyRow = nullptr;
              return row;
          }
      private:
          unsigned hash;
          const void *keyRow;
      };
  
      static CClassMeta<HashDedupElement> hashDedupElementMeta;
  
      class HashDedupTable : public SuperHashTable
      {
      public:
          HashDedupTable(IHThorHashDedupArg & _helper, CRoxieServerHashDedupActivity & _activity)
              : helper(_helper), 
                keySize(helper.queryKeySize()),
                activity(_activity)
          {
              bestCompare=helper.queryCompareBest();
          }
          virtual ~HashDedupTable()
          {
              //elementRowAllocator is a unique allocator, so all rows can be freed in a single call
              if (elementRowAllocator)
                  elementRowAllocator->releaseAllRows();
              tablecount = 0;
          }
  
          virtual unsigned getHashFromElement(const void *et) const       
          {
              const HashDedupElement *element = reinterpret_cast<const HashDedupElement *>(et);
              return element->queryHash(); 
          }
          virtual unsigned getHashFromFindParam(const void *fp) const { throwUnexpected(); }
          virtual const void * getFindParam(const void *et) const { throwUnexpected(); }
          virtual bool matchesElement(const void *et, const void *searchET) const { throwUnexpected(); }
          virtual bool matchesFindParam(const void *et, const void *key, unsigned fphash) const 
          { 
              const HashDedupElement *element = reinterpret_cast<const HashDedupElement *>(et);
              if (fphash != element->queryHash())
                  return false;
              return (helper.queryKeyCompare()->docompare(element->queryRow(), key) == 0); 
          }
          virtual void onAdd(void *et) {}
          virtual void onRemove(void *et)
          {
              ReleaseRoxieRow(et);
          }
  
          void onCreate(IRoxieAgentContext *ctx)
          {
              keyRowAllocator.setown(activity.createRowAllocator(keySize.queryOriginal()));
              elementRowAllocator.setown(activity.createRowAllocatorEx(&hashDedupElementMeta, roxiemem::RHFunique|roxiemem::RHFscanning|roxiemem::RHFdelayrelease));
          }
  
          void reset()
          {
              kill(); 
          }
  
          bool insert(const void * row)
          {
              unsigned hash = helper.queryHash()->hash(row);
              RtlDynamicRowBuilder keyRowBuilder(keyRowAllocator, true);
              size32_t thisKeySize = helper.recordToKey(keyRowBuilder, row);
              OwnedConstRoxieRow keyRow = keyRowBuilder.finalizeRowClear(thisKeySize);
              if (find(hash, keyRow.get()))
                  return false;
              addNew(createElement(hash, keyRow.getClear()), hash);
              return true;
          }
  
          bool insertBest(const void * nextrow)
          {
              unsigned hash = helper.queryHash()->hash(nextrow);
              const void *et = find(hash, nextrow);
              if (et)
              {
                  const HashDedupElement *element = reinterpret_cast<const HashDedupElement *>(et);
                  const void * row = element->queryRow();
                  if (bestCompare->docompare(row,nextrow) <= 0)
                      return false;
                  removeExact( const_cast<void *>(et));
                  // drop-through to add new row
              }
              addNew(createElement(hash, nextrow), hash);
              return true;
          }
  
          void gatherStats(CRuntimeStatisticCollection & merged) const
          {
              if (keyRowAllocator)
                  keyRowAllocator->gatherStats(merged);
          }
  
      private:
          inline void *createElement(unsigned hash, const void *row)
          {
              return elementRowAllocator->finalizeRow(sizeof(HashDedupElement), new (elementRowAllocator->createRow()) HashDedupElement(hash, row), sizeof(HashDedupElement));
          }
          IHThorHashDedupArg & helper;
          CachedOutputMetaData keySize;
          Owned<IEngineRowAllocator> keyRowAllocator;
          Owned<IEngineRowAllocator> elementRowAllocator;
          CRoxieServerHashDedupActivity & activity;
          ICompare * bestCompare;
      } table;
  
  public:
      CRoxieServerHashDedupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorHashDedupArg &)basehelper), table(helper, *this), hashTableFilled(false), hashDedupTableIter(table)
      {
          eof = false;
          keepBest = helper.keepBest();
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(colocalParent);
          table.onCreate(ctx);
      }
  
      virtual void reset()
      {
          table.reset();
          eof = false;
          CRoxieServerActivity::reset();
          hashTableFilled = false;
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (keepBest)
          {
              if (eof)
                  return NULL;
              // Populate hash table with best rows
              if (!hashTableFilled)
              {
                  const void * next = inputStream->nextRow();
                  while(next)
                  {
                      if (!table.insertBest(next))
                          ReleaseRoxieRow(next);
                      next = inputStream->nextRow();
                  }
                  if (table.count() == 0)
                      eof = true;
                  hashTableFilled = true;
                  hashDedupTableIter.first();
              }
  
              // Iterate throw hash table returning rows
              if (hashDedupTableIter.isValid())
              {
                  HashDedupElement &el = hashDedupTableIter.query();
  
                  const void * row = el.getRowClear();
                  hashDedupTableIter.next();
                  return row;
              }
              table.reset();
              hashTableFilled = false;
              return NULL;
          }
          else
          {
              while(!eof)
              {
                  const void * next = inputStream->nextRow();
                  if(!next)
                  {
                      if (table.count() == 0)
                          eof = true;
                      table.reset();
                      return NULL;
                  }
  
                  if(table.insert(next))
                      return next;
                  else
                      ReleaseRoxieRow(next);
              }
              return NULL;
          }
      }
  private:
      bool keepBest;
      bool hashTableFilled;
      SuperHashIteratorOf<HashDedupElement> hashDedupTableIter;
  };
  
  CClassMeta<CRoxieServerHashDedupActivity::HashDedupElement> CRoxieServerHashDedupActivity::hashDedupElementMeta;
  
  class CRoxieServerHashDedupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerHashDedupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerHashDedupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerHashDedupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerHashDedupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerRollupActivity : public CRoxieServerActivity
  {
      IHThorRollupArg &helper;
      OwnedConstRoxieRow left;
      OwnedConstRoxieRow prev;
      OwnedConstRoxieRow right;
      bool readFirstRow;
  
  public:
      CRoxieServerRollupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorRollupArg &)basehelper)
      {
          readFirstRow = false;
      }
  
      ~CRoxieServerRollupActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          readFirstRow = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()    
      {
          left.clear();
          prev.clear();
          right.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!readFirstRow)
          {
              left.setown(inputStream->nextRow());
              prev.set(left);
              readFirstRow = true;
          }
  
          for (;;)
          {
              right.setown(inputStream->nextRow());
              if(!prev || !right || !helper.matches(prev,right))
              {
                  const void * ret = left.getClear();
                  if(ret)
                      processed++;
                  left.setown(right.getClear());
                  prev.set(left);
                  return ret;
              }
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  unsigned outSize = helper.transform(rowBuilder, left, right);
                  if (outSize)
                      left.setown(rowBuilder.finalizeRowClear(outSize));
  
                  if (helper.getFlags() & RFrolledismatchleft)
                      prev.set(left);
                  else
                      prev.set(right);
              }
              catch(IException * E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerRollupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerRollupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerRollupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerRollupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerRollupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNormalizeActivity : public CRoxieServerActivity
  {
      IHThorNormalizeArg &helper;
      unsigned numThisRow;
      unsigned curRow;
      const void *buffer;
      unsigned numProcessedLastGroup;
  
  public:
      CRoxieServerNormalizeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorNormalizeArg &)basehelper)
      {
          buffer = NULL;
          numThisRow = 0;
          curRow = 0;
          numProcessedLastGroup = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numThisRow = 0;
          curRow = 0;
          numProcessedLastGroup = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          ReleaseClearRoxieRow(buffer);
          CRoxieServerActivity::reset();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              while (curRow == numThisRow)
              {
                  if (buffer)
                      ReleaseClearRoxieRow(buffer);
                  buffer = inputStream->nextRow();
                  if (!buffer && (processed == numProcessedLastGroup))
                      buffer = inputStream->nextRow();
                  if (!buffer)
                  {
                      numProcessedLastGroup = processed;
                      return NULL;
                  }
  
                  curRow = 0;
                  numThisRow = helper.numExpandedRows(buffer);
              }
  
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  unsigned actualSize = helper.transform(rowBuilder, buffer, ++curRow);
                  if (actualSize != 0)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(actualSize);
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  
  };
  
  class CRoxieServerNormalizeActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNormalizeActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNormalizeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNormalizeChildActivity : public CRoxieServerActivity
  {
      IHThorNormalizeChildArg &helper;
      unsigned numThisRow;
      unsigned curRow;
      const void *buffer;
      unsigned numProcessedLastGroup;
      INormalizeChildIterator * cursor;
      const void * curChildRow;
  
      bool advanceInput()
      {
          for (;;)
          {
              ReleaseClearRoxieRow(buffer);
              buffer = inputStream->nextRow();
              if (!buffer && (processed == numProcessedLastGroup))
                  buffer = inputStream->nextRow();
              if (!buffer)
              {
                  numProcessedLastGroup = processed;
                  return false;
              }
  
              curChildRow = cursor->first(buffer);
              if (curChildRow)
              {
                  curRow = 0;
                  return true;
              }
          }
      }
  
  public:
      CRoxieServerNormalizeChildActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorNormalizeChildArg &)basehelper)
      {
          buffer = NULL;
          cursor = NULL;
          numThisRow = 0;
          curRow = 0;
          numProcessedLastGroup = 0;
          curChildRow = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numThisRow = 0;
          curRow = 0;
          numProcessedLastGroup = 0;
          curChildRow = NULL;
  
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          cursor = helper.queryIterator();
      }
  
      virtual void stop()
      {
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          cursor = NULL;
          ReleaseClearRoxieRow(buffer);
          CRoxieServerActivity::reset(); 
      }
  
      virtual bool needsAllocator() const { return true; }
  
      const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if (!buffer)
              {
                  if (!advanceInput())
                      return NULL;
              }
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, buffer, curChildRow, ++curRow);
                  curChildRow = cursor->next();
                  if (!curChildRow)
                      ReleaseClearRoxieRow(buffer);
                  if (outSize != 0)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerNormalizeChildActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerNormalizeChildActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNormalizeChildActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNormalizeChildActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNormalizeChildActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNormalizeLinkedChildActivity : public CRoxieServerActivity
  {
      IHThorNormalizeLinkedChildArg &helper;
      OwnedConstRoxieRow curParent;
      OwnedConstRoxieRow curChild;
      unsigned numProcessedLastGroup;
  
      bool advanceInput()
      {
          for (;;)
          {
              curParent.setown(inputStream->nextRow());
              if (!curParent && (processed == numProcessedLastGroup))
                  curParent.setown(inputStream->nextRow());
              if (!curParent)
              {
                  numProcessedLastGroup = processed;
                  return false;
              }
  
              curChild.set(helper.first(curParent));
              if (curChild)
                  return true;
          }
      }
  
  public:
      CRoxieServerNormalizeLinkedChildActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorNormalizeLinkedChildArg &)basehelper)
      {
          numProcessedLastGroup = 0;
      }
      ~CRoxieServerNormalizeLinkedChildActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          curParent.clear();
          curChild.clear();
          CRoxieServerActivity::reset(); 
      }
  
      const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if (!curParent)
              {
                  if (!advanceInput())
                      return NULL;
              }
              try
              {
                  const void *ret = curChild.getClear();
                  curChild.set(helper.next());
                  if (!curChild)
                      curParent.clear();
                  if (ret)
                  {
                      processed++;
                      return ret;
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerNormalizeLinkedChildActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerNormalizeLinkedChildActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNormalizeLinkedChildActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNormalizeLinkedChildActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNormalizeLinkedChildActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSortActivity : public CRoxieServerActivity
  {
  protected:
      IHThorSortArg &helper;
      ICompare *compare;
      Owned<ISortAlgorithm> sorter;
      bool readInput;
      RoxieSortAlgorithm sortAlgorithm;
      unsigned sortFlags;
  
  public:
      CRoxieServerSortActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, RoxieSortAlgorithm _sortAlgorithm, unsigned _sortFlags)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSortArg &)basehelper), sortAlgorithm(_sortAlgorithm), sortFlags(_sortFlags)
      {
          compare = helper.queryCompare();
          readInput = false;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          if (sortAlgorithm==unknownSortAlgorithm)
              sorter.clear();
          else
              sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          assertex(!readInput);
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          if (sorter)
              sorter->reset();
          readInput = false;
          CRoxieServerActivity::reset();
      }
  
      static RoxieSortAlgorithm useAlgorithm(const char *algorithmName, unsigned sortFlags)
      {
          RoxieSortAlgorithm sortAlgorithm = unknownSortAlgorithm;
          if (!algorithmName)
              algorithmName = "";
          if (stricmp(algorithmName, "quicksort")==0)
          {
              switch (sortFlags & (TAFstable|TAFspill))
              {
              case 0: sortAlgorithm = quickSortAlgorithm; break;
              case TAFstable: sortAlgorithm = stableQuickSortAlgorithm; break;
              case TAFspill: sortAlgorithm = spillingQuickSortAlgorithm; break;
              case TAFstable|TAFspill: sortAlgorithm = stableSpillingQuickSortAlgorithm; break;
              }
          }
          else if (stricmp(algorithmName, "parquicksort")==0)
          {
              switch (sortFlags & TAFstable)
              {
              case 0: sortAlgorithm = parallelQuickSortAlgorithm; break;
              case TAFstable: sortAlgorithm = parallelStableQuickSortAlgorithm; break;
              default: throwUnexpected();
              }
          }
          else if (stricmp(algorithmName, "heapsort")==0)
              sortAlgorithm = heapSortAlgorithm; // NOTE - we do allow UNSTABLE('heapsort') in order to facilitate runtime selection. Also explicit selection of heapsort overrides request to spill
          else if (stricmp(algorithmName, "mergesort")==0)
              sortAlgorithm = (sortFlags & TAFspill) ? spillingMergeSortAlgorithm : mergeSortAlgorithm;
          else if (stricmp(algorithmName, "parmergesort")==0)
              sortAlgorithm = (sortFlags & TAFspill) ? spillingParallelMergeSortAlgorithm : parallelMergeSortAlgorithm;
  #ifdef _USE_TBB
          else if (stricmp(algorithmName, "tbbqsort")==0)
              sortAlgorithm = tbbQuickSortAlgorithm;
          else if (stricmp(algorithmName, "tbbstableqsort")==0)
              sortAlgorithm = tbbStableQuickSortAlgorithm;
  #endif
          else
          {
              if (*algorithmName)
                  OWARNLOG(ROXIE_UNKNOWN_ALGORITHM, "Ignoring unsupported sort order algorithm '%s', using default", algorithmName);
              if (sortFlags & TAFspill)
                  sortAlgorithm = ((sortFlags & TAFunstable) != 0) ? spillingQuickSortAlgorithm : stableSpillingQuickSortAlgorithm;
              else if (sortFlags & TAFunstable)
                  sortAlgorithm = quickSortAlgorithm;
              else
                  sortAlgorithm = heapSortAlgorithm;
          }
          return sortAlgorithm;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!readInput)
          {
              if (sortAlgorithm == unknownSortAlgorithm)
              {
                  sorter.clear();
                  OwnedRoxieString algorithmName(helper.getAlgorithm());
                  sortAlgorithm = useAlgorithm(algorithmName, sortFlags);
                  sorter.setown(createSortAlgorithm(sortAlgorithm, compare, ctx->queryRowManager(), meta, ctx->queryCodeContext(), tempDirectory, activityId));
              }
              sorter->prepare(inputStream);
              noteStatistic(StTimeSortElapsed, cycle_to_nanosec(sorter->getElapsedCycles(true)));
              readInput = true;
          }
          const void *ret = sorter->next();
          if (ret)
              processed++;
          else
          {
              sorter->reset();
              readInput = false; // ready for next group
          }
          return ret;
      }
  };
  
  class CRoxieServerSortActivityFactory : public CRoxieServerActivityFactory
  {
      RoxieSortAlgorithm sortAlgorithm;
      unsigned sortFlags;
  public:
      CRoxieServerSortActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, sortStatistics)
      {
          Owned<IHThorSortArg> sortHelper = (IHThorSortArg *) helperFactory();
          const char *algorithmName = NULL;
          sortFlags = sortHelper->getAlgorithmFlags();
          if (sortFlags & TAFconstant)
              algorithmName = sortHelper->getAlgorithm();
          bool forceSpill = _queryFactory.queryOptions().allSortsMaySpill || _graphNode.getPropBool("hint[@name='spill']/@value", false);
          if (forceSpill)
              sortFlags |= TAFspill;
          if (!(sortFlags & TAFunstable))
              sortFlags |= TAFstable;    // Assume stable unless specifically requested otherwise
          if (sortFlags & TAFconstant)
              sortAlgorithm = CRoxieServerSortActivity::useAlgorithm(algorithmName, sortFlags);
          else
              sortAlgorithm = unknownSortAlgorithm;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSortActivity(_ctx, this, _probeManager, sortAlgorithm, sortFlags);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSortActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSortActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  static int compareUint64(const void * _left, const void * _right)
  {
      const unsigned __int64 left = *static_cast<const unsigned __int64 *>(_left);
      const unsigned __int64 right = *static_cast<const unsigned __int64 *>(_right);
      if (left < right)
          return -1;
      if (left > right)
          return -1;
      return 0;
  }
  
  class CRoxieServerQuantileActivity : public CRoxieServerActivity
  {
  protected:
      Owned<ISortAlgorithm> sorter;
      IHThorQuantileArg &helper;
      ConstPointerArray sorted;
      ICompare *compare;
      unsigned flags;
      double skew;
      unsigned __int64 numDivisions;
      bool rangeIsAll;
      size32_t rangeSize;
      rtlDataAttr rangeValues;
      bool calculated;
      bool processedAny;
      bool anyThisGroup;
      bool eof;
      unsigned curQuantile;
      unsigned curIndex;
      unsigned curIndexExtra;
      unsigned skipSize;
      unsigned skipExtra;
      unsigned prevIndex;
  
  public:
      CRoxieServerQuantileActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _flags)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorQuantileArg &)basehelper), flags(_flags)
      {
          compare = helper.queryCompare();
          skew = 0.0;
          numDivisions = 0;
          calculated = false;
          processedAny = false;
          anyThisGroup = false;
          eof = false;
          rangeIsAll = true;
          rangeSize = 0;
          curQuantile = 0;
          prevIndex = 0;
          curIndex = 0;
          curIndexExtra = 0;
          skipSize = 0;
          skipExtra = 0;
          if (flags & TQFunstable)
              sorter.setown(createQuickSortAlgorithm(compare));
          else
              sorter.setown(createMergeSortAlgorithm(compare));
      }
  
      virtual void reset()
      {
          sorter->reset();
          calculated = false;
          processedAny = false;
          anyThisGroup = false;
          ReleaseRoxieRows(sorted);
          CRoxieServerActivity::reset();
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          skew = helper.getSkew();
          numDivisions = helper.getNumDivisions();
          //Check for -ve integer values and treat as out of range
          if ((__int64)numDivisions < 1)
              numDivisions = 1;
          if (flags & TQFhasrange)
              helper.getRange(rangeIsAll, rangeSize, rangeValues.refdata());
          else
          {
              rangeIsAll = true;
              rangeSize = 0;
          }
          if (rangeSize)
          {
              //Sort the range items into order to allow quick comparison
              qsort(rangeValues.getdata(), rangeSize / sizeof(unsigned __int64), sizeof(unsigned __int64), compareUint64);
          }
          calculated = false;
          processedAny = false;
          anyThisGroup = false;
          eof = false;
          curQuantile = 0;
          prevIndex = 0;
          curIndex = 0;
          curIndexExtra = 0;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          const void * ret = NULL;
          for(;;)
          {
              if (!calculated)
              {
                  if (flags & TQFlocalsorted)
                  {
                      for (;;)
                      {
                          const void * next = inputStream->nextRow();
                          if (!next)
                              break;
                          sorted.append(next);
                      }
                  }
                  else
                  {
                      sorter->prepare(inputStream);
                      sorter->getSortedGroup(sorted);
                  }
  
                  if (sorted.ordinality() == 0)
                  {
                      if (processedAny)
                      {
                          eof = true;
                          return NULL;
                      }
  
                      //Unusual case 0 rows - add a default row instead
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      size32_t thisSize = helper.createDefault(rowBuilder);
                      sorted.append(rowBuilder.finalizeRowClear(thisSize));
                  }
  
                  calculated = true;
                  processedAny = true;
                  anyThisGroup = false;
                  curQuantile = 0;
                  curIndex = 0;
                  curIndexExtra = (unsigned)((numDivisions-1) / 2);   // to ensure correctly rounded up
                  prevIndex = curIndex-1; // Ensure it doesn't match
                  skipSize = (unsigned)(sorted.ordinality() / numDivisions);
                  skipExtra = (unsigned)(sorted.ordinality() % numDivisions);
              }
  
              if (isQuantileIncluded(curQuantile))
              {
                  if (!(flags & TQFdedup) || (prevIndex != curIndex))
                  {
                      const void * lhs = sorted.item(curIndex);
                      if (flags & TQFneedtransform)
                      {
                          unsigned outSize;
                          RtlDynamicRowBuilder rowBuilder(rowAllocator);
                          try
                          {
                              outSize = helper.transform(rowBuilder, lhs, curQuantile);
                          }
                          catch (IException *E)
                          {
                              throw makeWrappedException(E);
                          }
  
                          if (outSize)
                              ret = rowBuilder.finalizeRowClear(outSize);
                      }
                      else
                      {
                          LinkRoxieRow(lhs);
                          ret = lhs;
                      }
                      prevIndex = curIndex; // even if output was skipped?
                  }
              }
  
              curIndex += skipSize;
              curIndexExtra += skipExtra;
              if (curIndexExtra >= numDivisions)
              {
                  curIndex++;
                  curIndexExtra -= numDivisions;
              }
              //Ensure the current index always stays valid.
              if (curIndex >= sorted.ordinality())
                  curIndex = sorted.ordinality()-1;
              curQuantile++;
              if (curQuantile > numDivisions)
              {
                  ReleaseRoxieRows(sorted);
                  sorter->reset();
                  calculated = false; // ready for next group
              }
  
              if (ret)
              {
                  anyThisGroup = true;
                  processed++;
                  return ret;
              }
  
              if (curQuantile > numDivisions)
              {
                  if (anyThisGroup)
                      return NULL;
              }
          }
      }
  
      bool isQuantileIncluded(unsigned quantile) const
      {
          if (quantile == 0)
              return (flags & TQFfirst) != 0;
          if (quantile == numDivisions)
              return (flags & TQFlast) != 0;
          if (rangeIsAll)
              return true;
          //Compare against the list of ranges provided
          //MORE: Since the list is sorted should only need to compare the next (and allow for dups)
          unsigned rangeNum = rangeSize / sizeof(unsigned __int64);
          const unsigned __int64 * ranges = static_cast<const unsigned __int64 *>(rangeValues.getdata());
          for (unsigned i = 0; i < rangeNum; i++)
          {
              if (ranges[i] == quantile)
                  return true;
          }
          return false;
      }
  };
  
  class CRoxieServerQuantileActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned flags;
  
  public:
      CRoxieServerQuantileActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorQuantileArg> quantileHelper = (IHThorQuantileArg *) helperFactory();
          flags = quantileHelper->getFlags();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerQuantileActivity(_ctx, this, _probeManager, flags);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerQuantileActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerQuantileActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerSortedActivity : public CRoxieServerActivity
  {
      IHThorSortedArg &helper;
      ICompare * compare;
      const void *prev; 
      IRangeCompare * stepCompare;
  
  public:
  
      CRoxieServerSortedActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSortedArg &)basehelper)
      {
          prev = NULL;
          compare = helper.queryCompare();
          stepCompare = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          IInputSteppingMeta * stepMeta = input->querySteppingMeta();
          if (stepMeta)
              stepCompare = stepMeta->queryCompare();
          prev = NULL;
      }
  
      virtual void reset()
      {
          ReleaseClearRoxieRow(prev);
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void *ret = inputStream->nextRow();
          if (ret && prev && compare->docompare(prev, ret) > 0)
          {
              // MORE - better to give mismatching rows that indexes?
              throw MakeStringException(ROXIE_NOT_SORTED, "SORTED(%u) detected incorrectly sorted rows  (row %d,  %d))", activityId, processed, processed+1);
          }
          ReleaseRoxieRow(prev);
          prev = ret;
          if (ret)
          {
              LinkRoxieRow(prev);
              processed++;
          }
          return ret;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void *ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (ret && prev && compare->docompare(prev, ret) > 0)
          {
              // MORE - better to give mismatching rows that indexes?
              throw MakeStringException(ROXIE_NOT_SORTED, "SORTED(%u) detected incorrectly sorted rows  (row %d,  %d))", activityId, processed, processed+1);
          }
          ReleaseRoxieRow(prev);
          prev = ret;
          if (ret)
          {
              LinkRoxieRow(prev);
              processed++;
          }
          return ret;
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          inputStream->resetEOF();
      }
  };
  
  class CRoxieServerSortedActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSortedActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSortedActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSortedActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSortedActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerThroughSpillActivity : public CRoxieServerActivity
  {
  
      /*
          BE VERY CAREFUL - this code is tricky.
          Note that starts and stops (and resets) can occur in strange orders
          The FIRST start OR stop must initialize the activity but only the first START should call the upstream start.
          The last stop should call the upstream stop.
          The first reset should call the upstream reset.
  
          The calculation of whether a row is needed for other (yet to come) outputs needs to work correctly even if the output in question has 
          not yet had start or stop called - for this to happen init() is called on all outputs on the first start or stop.
  
          Some outputs may be completely pruned away when used in a GRAPH - these outputs should not receive any start/stop/reset and should be
          ignored in the minIndex calculation
  
      */
  
  public:
      IHThorArg &helper;
      unsigned activeOutputs;
      unsigned numOutputs;
      unsigned numOriginalOutputs;
  
      QueueOf<const void, true> buffer;
      CriticalSection crit;
      CriticalSection crit2;
      unsigned tailIdx;
      unsigned headIdx;
      Owned<IException> readError;
      Owned<IException> startError;
  
      class OutputAdaptor : implements IEngineRowStream, implements IFinalRoxieInput, public CInterface
      {
          bool eof, eofpending, stopped;
  
      public:
          CRoxieServerThroughSpillActivity *parent;
          unsigned idx;
          unsigned oid;
          unsigned processed = 0;
          unsigned numStarts = 0;
          cycle_t totalCycles;  // We track this per output so that the pullers get a meaningful value to use when calculating their localtime
  
      public:
          IMPLEMENT_IINTERFACE;
  
          OutputAdaptor()
          {
              parent = NULL;
              oid = 0;
              idx = 0;
              processed = 0;
              totalCycles = 0;
              eofpending = false;
              eof = false;
              stopped = false;
          }
  
          ~OutputAdaptor()
          {
              if (traceStartStop)
                  DBGLOG("%p ~OutputAdaptor %d", this, oid);
          }
  
          void init()
          {
              if (traceStartStop)
                  DBGLOG("%p init Input adaptor %d", this, oid);
              idx = 0;
              eofpending = false;
              eof = false;
              stopped = false;
          }
  
          virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(idx==0); return this; }
          virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
  
          virtual IRoxieServerActivity *queryActivity()
          {
              return parent;
          }
  
          virtual IIndexReadActivityInfo *queryIndexReadActivity()
          {
              return parent->queryIndexReadActivity();
          }
  
          virtual unsigned __int64 queryTotalCycles() const
          {
              return totalCycles;
          }
  
          virtual const void * nextRow()
          {
              SimpleActivityTimer t(totalCycles, parent->timeActivities);
              if (eof)
                  return NULL;
              const void *ret = parent->readBuffered(idx, oid);
  #ifdef TRACE_SPLIT
              parent->CTXLOG("Adaptor %d got back %p for record %d", oid, ret, idx);
  #endif
              if (ret)
              {
                  processed++;
                  eofpending = false;
              }
              else if (eofpending)
                  eof = true;
              else
                  eofpending = true;
              return ret;
          }
  
          virtual IOutputMetaData * queryOutputMeta() const
          {
              return parent->queryOutputMeta();
          }
  
          virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
          {
              // NOTE: it is tempting to move the init() of all output adaptors here. However that is not a good idea, 
              // since adaptors that have not yet started or stopped (but are going to) still need to have been init()'ed
              // for minIndex to give the correct answers
              // therefore, we call init() on all adaptors on receipt of the first start() or stop()
  
              if (traceStartStop)
                  parent->CTXLOG("%p start Input adaptor %d stopped = %d", this, oid, stopped);
              parent->start(oid, parentExtractSize, parentExtract, paused);
              numStarts++;
          }
  
          virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
          {
              parent->connectInputStreams(consumerOrdered);
              streams.append(this);
              return NULL;
          }
  
          virtual void stop()
          {
              if (traceStartStop)
                  parent->CTXLOG("%p stop Input adaptor %d stopped = %d", this, oid, stopped);
              if (!stopped)
              {
                  parent->stop(oid, idx);   // NOTE - may call init()
                  stopped = true; // parent code relies on stop being called exactly once per adaptor, so make sure it is!
              }
          };
  
          virtual void reset()
          {
              if (traceStartStop)
                  parent->CTXLOG("%p reset Input adaptor %d stopped = %d", this, oid, stopped);
              parent->reset(oid);
              idx = 0; // value should not be relevant really but this is the safest...
              stopped = false;
          };
  
          virtual void resetEOF()
          {
              parent->resetEOF();
          }
  
          virtual void checkAbort()
          {
              parent->checkAbort();
          }
      } *adaptors;
      bool *used;
  
      unsigned minIndex(unsigned exceptOid)
      {
          // MORE - yukky code (and slow). Could keep them heapsorted by idx or something
          // this is trying to determine whether any of the adaptors will in the future read a given record
          unsigned minIdx = (unsigned) -1;
          for (unsigned i = 0; i < numOriginalOutputs; i++)
          {
              if (i != exceptOid && used[i] && adaptors[i].idx < minIdx)
                  minIdx = adaptors[i].idx;
          }
          return minIdx;
      }
  
      inline bool isLastTailReader(unsigned exceptOid)
      {
          for (unsigned i = 0; i < numOriginalOutputs; i++)
          {
              if (i != exceptOid && adaptors[i].idx == tailIdx && used[i])
                  return false;
          }
          return true;
      }
  
      void initOutputs()
      {
          activeOutputs = numOutputs;
          for (unsigned i = 0; i < numOriginalOutputs; i++)
              if (used[i])
                  adaptors[i].init();
          state = STATEstarting;
      }
  
  public:
      CRoxieServerThroughSpillActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numOutputs)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper(basehelper), numOutputs(_numOutputs)
      {
          numOriginalOutputs = numOutputs;
          adaptors = new OutputAdaptor[numOutputs];
          used = new bool[numOutputs];
          for (unsigned i = 0; i < numOutputs; i++)
          {
              adaptors[i].parent = this;
              adaptors[i].oid = i;
              used[i] = false;
          }
          tailIdx = 0;
          headIdx = 0;
          activeOutputs = numOutputs;
      }
  
      ~CRoxieServerThroughSpillActivity()
      {
          delete [] adaptors;
          delete [] used;
      }
  
      const void *readBuffered(unsigned &idx, unsigned oid)
      {
          //False positives are fine, false negatives are not.. so headIdx must only be updated when a row will be available.
          const unsigned curIdx = idx;
          if (curIdx == headIdx) // test once without getting the crit2 sec
          {
              CriticalBlock b2(crit2);  // but only one puller gets to read the head
              if (curIdx == headIdx) // test again now that we have it
              {
                  ActivityTimer t(activityStats, timeActivities); // NOTE - time spent waiting for crit not included here. But it will have been included on the totalTime of the person holding the crit, so that is right
                  if (readError)
                      throw readError.getLink();  // Read exceptions are only throw if you try to read past the row where they happened
  
                  try
                  {
                      const void *row = inputStream->nextRow();
                      if (activeOutputs==1)
                      {
  #ifdef TRACE_SPLIT
                          CTXLOG("spill %d optimised return of %p", activityId, row);
  #endif
                          headIdx++;
                          idx++;
                          return row;  // optimization for the case where only one output still active.
                      }
                      CriticalBlock b(crit);
                      headIdx++;
                      idx++;
                      if (activeOutputs==1)
                          return row;  // optimization for the case where only one output still active.
  
                      buffer.enqueue(row);
                      if (row) LinkRoxieRow(row);
                      return row;
                  }
                  catch (IException *E)
                  {
  #ifdef TRACE_SPLIT
                      CTXLOG("spill %d caught exception", activityId);
  #endif
                      readError.set(E);
                      throw;
                  }
                  catch (...)
                  {
                      IException *E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught in CRoxieServerThroughSpillActivity::readBuffered");
                      readError.set(E);
                      throw E;
                  }
              }
          }
  
          CriticalBlock b(crit);
          ActivityTimer t(activityStats, timeActivities); // NOTE - time spent waiting for crit not included here. But it will have been included on the totalTime of the person holding the crit, so that is right
          unsigned lidx = curIdx - tailIdx;
          idx++;
          if (!lidx)
          {
              // The numOutputs <= 2 check optimizes slightly the common case of 2-way splitters.
              // The rationale is that I MUST be the last reader if there are only 2 - the other guy must have put the record there in the first place
              if (numOutputs <= 2 || isLastTailReader(oid)) // NOTE - this includes the case where minIndex returns (unsigned) -1, meaning no other active pullers
              {
                  tailIdx++;
                  const void *ret = buffer.dequeue(); // no need to link - last puller
  #ifdef TRACE_SPLIT
                  CTXLOG("last puller return of %p", ret);
  #endif
                  return ret;
              }
          }
          const void *ret = buffer.item(lidx);
          if (ret) LinkRoxieRow(ret);
  #ifdef TRACE_SPLIT
          CTXLOG("standard return of %p", ret);
  #endif
          return ret;
      }
  
      virtual void start(unsigned oid, unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CriticalBlock b(crit);
          if (startError)
              throw startError.getLink();
          if (collectFactoryStatistics)
              factory->noteStarted(oid);
          if (traceStartStop)
              CTXLOG("SPLIT %p: start %d child %d activeOutputs %d numOutputs %d numOriginalOutputs %d state %s", this, activityId, oid, activeOutputs, numOutputs, numOriginalOutputs, queryStateText(state));
          if (state != STATEstarted)
          {
              if (state != STATEstarting)
                  initOutputs();
              tailIdx = 0;
              headIdx = 0;
              startError.clear();
              readError.clear();
              try
              {
                  CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
              }
              catch (IException *E)
              {
  #ifdef TRACE_SPLIT
                  CTXLOG("spill %d caught exception in start", activityId);
  #endif
                  startError.set(E);
                  throw;
              }
              catch (...)
              {
                  IException *E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught in CRoxieServerThroughSpillActivity::start");
                  startError.set(E);
                  throw E;
              }
          }
      }
  
      void stop(unsigned oid, unsigned & idx)
      {
          // Note that OutputAdaptor code ensures that stop is not called more than once per adaptor
          CriticalBlock b(crit);
  #ifdef TRACE_STARTSTOP
          if (traceStartStop)
          {
              CTXLOG("SPLIT %p: stop %d child %d activeOutputs %d numOutputs %d numOriginalOutputs %d state %s", this, activityId, oid, activeOutputs, numOutputs, numOriginalOutputs, queryStateText(state));
              if (watchActivityId && watchActivityId==activityId)
              {
                  CTXLOG("WATCH: stop %d", activityId);
              }
          }
  #endif
          if (state != STATEstarting && state != STATEstarted)
              initOutputs();
          if (activeOutputs > 1)
          {
              if (tailIdx==idx)
              {
                  // Discard all buffered rows that are there purely for this adaptor to read them
                  unsigned min = minIndex(oid);
                  if (min != (unsigned) -1) 
                  // what does -1 signify?? No-one wants anything? In which case can't we kill all rows?? 
                  // Should never happen though if there are still some active.
                  // there may be a small window where adaptors are blocked on the semaphore...
                  {
  #ifdef TRACE_SPLIT
                      CTXLOG("%p: Discarding buffered rows from %d to %d for oid %x (%d outputs active)", this, idx, min, oid, activeOutputs);
  #endif
                      while (tailIdx < min)
                      {
                          ReleaseRoxieRow(buffer.dequeue());
                          tailIdx++;
                      }
                  }
              }
              activeOutputs--;
              idx = (unsigned) -1; // causes minIndex not to save rows for me...
              return;
          }
  #ifdef TRACE_SPLIT
          CTXLOG("%p: All outputs done", this);
  #endif
          activeOutputs = numOutputs;
          CRoxieServerActivity::stop();
      };
  
      void reset(unsigned oid)
      {
          if (traceStartStop)
              CTXLOG("SPLIT %p: reset %d child %d activeOutputs %d numOutputs %d numOriginalOutputs %d state %s", this, activityId, oid, activeOutputs, numOutputs, numOriginalOutputs, queryStateText(state));
          activeOutputs = numOutputs;
          while (buffer.ordinality())
              ReleaseRoxieRow(buffer.dequeue());
          startError.clear();
          readError.clear();
          if (state != STATEreset) // make sure input is only reset once
              CRoxieServerActivity::reset();
      };
  
      void connectInputStreams(bool consumerOrdered)
      {
          if (!inputStream)
              CRoxieServerActivity::connectInputStreams(consumerOrdered);
      }
  
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // Internal logic error - we are not anybody's input
      }
  
      virtual IOutputMetaData * queryOutputMeta() const
      {
  //      if (outputMeta)
  //          return outputMeta;
  //      else
              return input->queryOutputMeta(); // not always known (e.g. disk write - though Gavin _could_ fill it in)
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          assertex(idx < numOriginalOutputs);
          assertex(!used[idx]);
          used[idx] = true;
          return &adaptors[idx];
      }
  
      virtual void resetOutputsUsed()
      {
          numOutputs = 1;
          activeOutputs = 1;
          // MORE RKC->GH should we be clearing the used array here? anywhere?
      }
  
      virtual void noteOutputUsed()
      {
          assertex(numOutputs < numOriginalOutputs);
          numOutputs++;
          activeOutputs = numOutputs;
      }
      virtual bool isPassThrough()
      {
          return numOutputs==1;
      }
  
      virtual void updateEdgeStats(IStatisticGatherer * statsBuilder) const
      {
          for (unsigned i = 0; i < numOutputs; i++)
              addEdgeStats(statsBuilder, i, adaptors[i].numStarts, adaptors[i].processed, 0);
      }
  
  };
  
  class CRoxieServerThroughSpillActivityFactory : public CRoxieServerMultiOutputFactory
  {
  public:
      CRoxieServerThroughSpillActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiOutputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorSpillArg> helper = (IHThorSpillArg *) helperFactory();
          setNumOutputs(helper->getTempUsageCount() + 1);
      }
  
      CRoxieServerThroughSpillActivityFactory(IQueryFactory &_queryFactory, HelperFactory *_helperFactory, unsigned _numOutputs, IPropertyTree &_graphNode)
          : CRoxieServerMultiOutputFactory(0, 0, _queryFactory, _helperFactory, TAKsplit, _graphNode, actStatistics)
      {
          setNumOutputs(_numOutputs);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerThroughSpillActivity(_ctx, this, _probeManager, numOutputs);
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerThroughSpillActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerThroughSpillActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  IRoxieServerActivityFactory *createRoxieServerThroughSpillActivityFactory(IQueryFactory &_queryFactory, HelperFactory *_factory, unsigned _numOutputs, IPropertyTree &_graphNode)
  {
      return new CRoxieServerThroughSpillActivityFactory(_queryFactory, _factory, _numOutputs, _graphNode);
  }
  
  //----------------------------------------------------------------------------------------------
  
  class CRoxieServerSplitActivityFactory : public CRoxieServerMultiOutputFactory
  {
  public:
      CRoxieServerSplitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiOutputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorSplitArg> helper = (IHThorSplitArg *) helperFactory();
          setNumOutputs(helper->numBranches());
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerThroughSpillActivity(_ctx, this, _probeManager, numOutputs);
      }
  
      virtual void gatherEdgeStats(IStatisticGatherer &builder, int channel, bool reset) const override
      {
          for (unsigned idx = 0; idx < numOutputs; idx++)
          {
              StatsEdgeScope e(builder, id, idx);
              edgestatsArray[idx].recordStatistics(builder, reset);
          }
      }
      // MORE - activityMetrics should probably put something, though the format is not especially amenable to multiple edges
  };
  
  IRoxieServerActivityFactory *createRoxieServerSplitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSplitActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  
  //=====================================================================================================
  
  #define PIPE_BUFSIZE 0x8000
  
  static IException *createPipeFailureException(const char *cmd, unsigned retcode, IPipeProcess *pipe)
  {
      StringBuffer msg;
      if(pipe->hasError())
      {
          try
          {
              char error[512];
              size32_t sz = pipe->readError(sizeof(error), error);
              if(sz && sz!=(size32_t)-1)
                  msg.append(", stderr: '").append(sz, error).append("'");
          }
          catch (IException *e)
          {
              EXCLOG(e, "Error reading pipe stderr");
              e->Release();
          }
      }
      return MakeStringException(ROXIE_PIPE_ERROR, "Pipe process %s returned error %u%s", cmd, retcode, msg.str());
  }
  
  class CRoxieServerPipeReadActivity : public CRoxieServerActivity
  {
      IHThorPipeReadArg &helper;
      Owned<IPipeProcess> pipe;
      StringAttr pipeCommand;
      Owned<IOutputRowDeserializer> rowDeserializer;
      Owned<IReadRowStream> readTransformer;
      bool groupSignalled;
  public:
      CRoxieServerPipeReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorPipeReadArg &)basehelper)
      {
          groupSignalled = true;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          rowDeserializer.setown(rowAllocator->createDiskDeserializer(ctx->queryCodeContext()));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          groupSignalled = true; // i.e. don't start with a NULL row
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (!readTransformer)
          {
              OwnedRoxieString xmlIteratorPath(helper.getXmlIteratorPath());
              readTransformer.setown(createReadRowStream(rowAllocator, rowDeserializer, helper.queryXmlTransformer(), helper.queryCsvTransformer(), xmlIteratorPath, helper.getPipeFlags()));
          }
          OwnedRoxieString pipeProgram(helper.getPipeProgram());
          openPipe(pipeProgram);
      }
  
      virtual void stop()
      {
          CRoxieServerActivity::stop();
          pipe.clear();
          readTransformer->setStream(NULL);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          while (!waitForPipe())
          {
              if (!pipe)
                  return NULL;
              if (helper.getPipeFlags() & TPFgroupeachrow)
              {
                  if (!groupSignalled)
                  {
                      groupSignalled = true;
                      return NULL;
                  }
              }
          }
          const void *ret = readTransformer->next();
          assertex(ret != NULL); // if ret can ever be NULL then we need to recode this logic
          processed++;
          groupSignalled = false;
          return ret;
      }
  
  protected:
      bool waitForPipe()
      {
          if (!pipe)
              return false;  // done
          if (!readTransformer->eos())
              return true;
          verifyPipe();
          return false;
      }
  
      void openPipe(char const * cmd)
      {
          pipeCommand.setown(cmd);
          pipe.setown(createPipeProcess());
          if(!pipe->run(NULL, cmd, ".", false, true, true, 0x10000))
          {
              // NB: pipe->run can't rely on the child process failing fast enough to return false here, failure picked up later with stderr context.
              WARNLOG(ROXIE_PIPE_ERROR, "Could not run pipe process %s", cmd);
          }
          Owned<ISimpleReadStream> pipeReader = pipe->getOutputStream();
          readTransformer->setStream(pipeReader.get());
      }
  
      void verifyPipe()
      {
          if (pipe)
          {
              unsigned err = pipe->wait();
              if(err && !(helper.getPipeFlags() & TPFnofail))
              {
                  throw createPipeFailureException(pipeCommand.get(), err, pipe);
              }
              pipe.clear();
          }
      }
  };
  
  class CRoxieServerPipeThroughActivity : public CRoxieServerActivity, implements IRecordPullerCallback
  {
      IHThorPipeThroughArg &helper;
      RecordPullerThread puller;
      Owned<IPipeProcess> pipe;
      StringAttr pipeCommand;
      InterruptableSemaphore pipeVerified;
      InterruptableSemaphore pipeOpened;
      CachedOutputMetaData inputMeta;
      Owned<IOutputRowSerializer> rowSerializer;
      Owned<IOutputRowDeserializer> rowDeserializer;
      Owned<IPipeWriteXformHelper> writeTransformer;
      Owned<IReadRowStream> readTransformer;
      bool firstRead;
      bool recreate;
      bool inputExhausted;
      bool groupSignalled;
  
  public:
  
      CRoxieServerPipeThroughActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorPipeThroughArg &)basehelper), puller(false)
      {
          recreate = helper.recreateEachRow();
          groupSignalled = true;
          firstRead = false;
          inputExhausted = false;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          rowSerializer.setown(inputMeta.createDiskSerializer(ctx->queryCodeContext(), activityId));
          rowDeserializer.setown(rowAllocator->createDiskDeserializer(ctx->queryCodeContext()));
          writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          firstRead = true;
          inputExhausted = false;
          groupSignalled = true; // i.e. don't start with a NULL row
          pipeVerified.reinit();
          pipeOpened.reinit();
          writeTransformer->ready();
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (!readTransformer)
          {
              OwnedRoxieString xmlIterator(helper.getXmlIteratorPath());
              readTransformer.setown(createReadRowStream(rowAllocator, rowDeserializer, helper.queryXmlTransformer(), helper.queryCsvTransformer(), xmlIterator, helper.getPipeFlags()));
          }
          if(!recreate)
          {
              OwnedRoxieString pipeProgram(helper.getPipeProgram());
              openPipe(pipeProgram);
          }
          puller.start(parentExtractSize, parentExtract, paused, 0, false, ctx);  // Pipe does not support preload presently - locks up
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (idx)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
          puller.setInput(this, _sourceIdx, _in);
          inputMeta.set(_in->queryOutputMeta());
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          puller.connectInputStreams(ctx, consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return puller.queryInput();
          else
              return NULL;
      }
  
      virtual void stop()
      {
          pipeVerified.interrupt(NULL);
          pipeOpened.interrupt(NULL);
          puller.stop();
          CRoxieServerActivity::stop();
          pipe.clear();
          if (readTransformer)
              readTransformer->setStream(NULL);
      }
  
      virtual void reset()
      {
          puller.reset();
          CRoxieServerActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          while (!waitForPipe())
          {
              if (!pipe)
                  return NULL;
              if (helper.getPipeFlags() & TPFgroupeachrow)
              {
                  if (!groupSignalled)
                  {
                      groupSignalled = true;
                      return NULL;
                  }
              }
          }
          const void *ret = readTransformer->next();
          assertex(ret != NULL); // if ret can ever be NULL then we need to recode this logic
          processed++;
          groupSignalled = false;
          return ret;
      }
  
      virtual void processRow(const void *row)
      {
          // called from puller thread
          if(recreate)
              openPipe(helper.getNameFromRow(row));
          writeTransformer->writeTranslatedText(row, pipe);
          ReleaseRoxieRow(row);
          if(recreate)
          {
              closePipe();
              pipeVerified.wait();
          }
      }
  
      virtual void processDone()
      {
          // called from puller thread
          if(recreate)
          {
              inputExhausted = true;
              pipeOpened.signal();
          }
          else
          {
              closePipe();
              pipeVerified.wait();
          }
      }
  
      virtual void processEOG()
      {
      }
  
      void processGroup(const ConstPointerArray &)
      {
          throwUnexpected();
      }
  
      virtual bool fireException(IException *e)
      {
          pipeOpened.interrupt(LINK(e));
          pipeVerified.interrupt(e);
          return true;
      }
  
  private:
      bool waitForPipe()
      {
          Owned<IException> pipeException;
          try
          {
              if (firstRead)
              {
                  pipeOpened.wait();
                  firstRead = false;
              }
              if (!pipe)
                  return false;  // done
              if (!readTransformer->eos())
                  return true;
          }
          catch (IException *e)
          {
              // NB: the original exception is probably a IPipeProcessException, but because InterruptableSemaphore rethrows it, we must catch it as an IException
              pipeException.setown(e);
          }
          verifyPipe();
          if (pipeException) // NB: verifyPipe may throw error based on pipe prog. output 1st.
              throw pipeException.getClear();
          if (recreate && !inputExhausted)
              pipeOpened.wait();
          return false;
      }
  
      void openPipe(char const * cmd)
      {
          pipeCommand.setown(cmd);
          pipe.setown(createPipeProcess());
          if(!pipe->run(NULL, cmd, ".", true, true, true, 0x10000))
          {
              // NB: pipe->run can't rely on the child process failing fast enough to return false here, failure picked up later with stderr context.
              WARNLOG(ROXIE_PIPE_ERROR, "Could not run pipe process %s", cmd);
          }
          writeTransformer->writeHeader(pipe);
          Owned<ISimpleReadStream> pipeReader = pipe->getOutputStream();
          readTransformer->setStream(pipeReader.get());
          pipeOpened.signal();
      }
  
      void closePipe()
      {
          writeTransformer->writeFooter(pipe);
          pipe->closeInput();
      }
  
      void verifyPipe()
      {
          if (pipe)
          {
              unsigned err = pipe->wait();
              if(err && !(helper.getPipeFlags() & TPFnofail))
              {
                  throw createPipeFailureException(pipeCommand.get(), err, pipe);
              }
              pipe.clear();
              pipeVerified.signal();
          }
      }
  };
  
  class CRoxieServerPipeWriteActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorPipeWriteArg &helper;
      Owned<IPipeProcess> pipe;
      StringAttr pipeCommand;
      CachedOutputMetaData inputMeta;
      Owned<IOutputRowSerializer> rowSerializer;
      Owned<IPipeWriteXformHelper> writeTransformer;
      bool firstRead;
      bool recreate;
      bool inputExhausted;
  public:
      CRoxieServerPipeWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorPipeWriteArg &)basehelper)
      {
          recreate = helper.recreateEachRow();
          firstRead = false;
          inputExhausted = false;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          inputMeta.set(input->queryOutputMeta());
          rowSerializer.setown(inputMeta.createDiskSerializer(ctx->queryCodeContext(), activityId));
          writeTransformer.setown(createPipeWriteXformHelper(helper.getPipeFlags(), helper.queryXmlOutput(), helper.queryCsvOutput(), rowSerializer));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          firstRead = true;
          inputExhausted = false;
          writeTransformer->ready();
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if(!recreate)
          {
              OwnedRoxieString pipeProgram(helper.getPipeProgram());
              openPipe(pipeProgram);
          }
      }
  
      virtual void stop()
      {
          if (!aborted && helper.getSequence() >= 0)
          {
              WorkunitUpdate wu = ctx->updateWorkUnit();
              if (wu)
              {
                  Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
                  if (result)
                  {
                      result->setResultTotalRowCount(processed);
                      result->setResultStatus(ResultStatusCalculated);
                  }
              }
          }
          CRoxieServerActivity::stop();
          pipe.clear();
      }
  
      virtual void onExecute()
      {
          Owned<IException> pipeException;
          try
          {
              for (;;)
              {
                  OwnedConstRoxieRow row(inputStream->ungroupedNextRow());
                  if (!row)
                      break;
                  processed++;
                  if(recreate)
                      openPipe(helper.getNameFromRow(row));
                  writeTransformer->writeTranslatedText(row, pipe);
                  if (recreate)
                  {
                      closePipe();
                      verifyPipe();
                  }
              }
              if (!recreate)
                  closePipe();
          }
          catch (IException *e)
          {
              pipeException.setown(e);
          }
          verifyPipe();
          if (pipeException) // NB: verifyPipe may throw error based on pipe prog. output 1st.
              throw pipeException.getClear();
      }
  
  private:
      void openPipe(char const * cmd)
      {
          pipeCommand.setown(cmd);
          pipe.setown(createPipeProcess());
          if(!pipe->run(NULL, cmd, ".", true, false, true, 0x10000))
          {
              // NB: pipe->run can't rely on the child process failing fast enough to return false here, failure picked up later with stderr context.
              WARNLOG(ROXIE_PIPE_ERROR, "Could not run pipe process %s", cmd);
          }
          writeTransformer->writeHeader(pipe);
      }
  
      void closePipe()
      {
          writeTransformer->writeFooter(pipe);
          pipe->closeInput();
      }
  
      void verifyPipe()
      {
          if (pipe)
          {
              unsigned err = pipe->wait();
              if(err && !(helper.getPipeFlags() & TPFnofail))
              {
                  throw createPipeFailureException(pipeCommand.get(), err, pipe);
              }
              pipe.clear();
          }
      }
  };
  
  class CRoxieServerPipeReadActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerPipeReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerPipeReadActivity(_ctx, this, _probeManager);
      }
  };
  
  class CRoxieServerPipeThroughActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerPipeThroughActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerPipeThroughActivity(_ctx, this, _probeManager);
      }
  };
  
  class CRoxieServerPipeWriteActivityFactory : public CRoxieServerInternalSinkFactory
  {
  public:
      CRoxieServerPipeWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
       : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerPipeWriteActivity(_ctx, this, _probeManager, usageCount);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerPipeReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerPipeReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  IRoxieServerActivityFactory *createRoxieServerPipeThroughActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerPipeThroughActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  IRoxieServerActivityFactory *createRoxieServerPipeWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
  {
      return new CRoxieServerPipeWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _isRoot);
  }
  //=====================================================================================================
  
  class CRoxieServerStreamedIteratorActivity : public CRoxieServerActivity
  {
      IHThorStreamedIteratorArg &helper;
      Owned<IRowStream> rows;
  
  public:
      CRoxieServerStreamedIteratorActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorStreamedIteratorArg &)basehelper)
      {
      }
  
      ~CRoxieServerStreamedIteratorActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          rows.setown(helper.createInput());
      }
  
      virtual void stop()
      {
          if (rows)
          {
              rows->stop();
              rows.clear();
          }
          CRoxieServerActivity::stop();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          assertex(rows != NULL);
          const void * next = rows->nextRow();
          if (next)
              processed++;
          return next;
      }
  
  };
  
  class CRoxieServerStreamedIteratorActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerStreamedIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerStreamedIteratorActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerStreamedIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerStreamedIteratorActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerFilterActivity : public CRoxieServerLateStartActivity
  {
      IHThorFilterArg &helper;
      bool anyThisGroup;
      IRangeCompare * stepCompare;
  
  public:
  
      CRoxieServerFilterActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerLateStartActivity(_ctx, _factory, _probeManager), helper((IHThorFilterArg &)basehelper)
      {
          anyThisGroup = false;
          stepCompare = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          anyThisGroup = false;
          CRoxieServerLateStartActivity::start(parentExtractSize, parentExtract, paused);
          lateStart(parentExtractSize, parentExtract, helper.canMatchAny());
  
          stepCompare = NULL;
          if (!eof)
          {
              IInputSteppingMeta * stepMeta = input->querySteppingMeta();
              if (stepMeta)
                  stepCompare = stepMeta->queryCompare();
          }
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          for (;;)
          {
              OwnedConstRoxieRow ret(inputStream->nextRow());
              if (!ret)
              {
                  //stop returning two NULLs in a row.
                  if (anyThisGroup)
                  {
                      anyThisGroup = false;
                      return NULL;
                  }
                  ret.setown(inputStream->nextRow());
                  if (!ret)
                  {
                      eof = true;
                      return NULL;                // eof...
                  }
              }
  
              if (helper.isValid(ret))
              {
                  anyThisGroup = true;
                  processed++;
                  return ret.getClear();
              }
          }
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          //Could assert that this isn't grouped
          // MORE - will need rethinking once we rethink the nextRowGE interface for global smart-stepping.
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          for (;;)
          {
              OwnedConstRoxieRow ret(inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra));
              if (!ret)
              {
                  eof = true;
                  return NULL;
              }
  
              if (!wasCompleteMatch)
              {
                  anyThisGroup = false; // RKC->GH - is this right??
                  return ret.getClear();
              }
  
              if (helper.isValid(ret))
              {
                  anyThisGroup = true;
                  processed++;
                  return ret.getClear();
              }
  
              if (!stepExtra.returnMismatches())
              {
                  ret.clear();
                  return nextRow();
              }
  
              //If asked to return mismatches we are only interested in mismatches that will force the stepped
              //condition to advance
              if (stepCompare->docompare(ret, seek, numFields) != 0)
              {
                  wasCompleteMatch = false;
                  anyThisGroup = false; // WHY?
                  return ret.getClear();
              }
          }
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          eof = prefiltered;
          anyThisGroup = false;
          inputStream->resetEOF();
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
  };
  
  class CRoxieServerFilterActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerFilterActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          optStableInput = false;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerFilterActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerFilterActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerFilterActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  class CRoxieServerFilterGroupActivity : public CRoxieServerLateStartActivity
  {
      IHThorFilterGroupArg &helper;
      unsigned curIndex;
      ConstPointerArray gathered;
      IRangeCompare * stepCompare;
  
  public:
  
      CRoxieServerFilterGroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerLateStartActivity(_ctx, _factory, _probeManager), helper((IHThorFilterGroupArg &)basehelper)
      {
          curIndex = 0;
          stepCompare = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerLateStartActivity::start(parentExtractSize, parentExtract, paused);
          lateStart(parentExtractSize, parentExtract, helper.canMatchAny());//sets eof
          assertex(eof == !helper.canMatchAny());
  
          curIndex = 0;
          stepCompare = NULL;
          if (!eof)
          {
              IInputSteppingMeta * inputStepping = input->querySteppingMeta();
              if (inputStepping)
                  stepCompare = inputStepping->queryCompare();
          }
      }
  
      virtual void reset()
      {
          releaseGathered();
          CRoxieServerLateStartActivity::reset();
      }
  
      inline void releaseGathered()
      {
          roxiemem::ReleaseRoxieRowRange(gathered.getArray(), curIndex, gathered.ordinality());
          gathered.kill();
          curIndex = 0;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if (eof)
                  return NULL;
  
              if (gathered.ordinality())
              {
                  if (gathered.isItem(curIndex))
                  {
                      const void * ret = gathered.item(curIndex++);
                      processed++;
                      return ret;
                  }
                  curIndex = 0;
                  gathered.kill();
                  return NULL;
              }
  
              const void * ret = inputStream->nextRow();
              while (ret)
              {
                  gathered.append(ret);
                  ret = inputStream->nextRow();
              }
  
              unsigned num = gathered.ordinality();
              if (num != 0)
              {
                  if (!helper.isValid(num, (const void * *)gathered.getArray()))
                      ReleaseRoxieRows(gathered);       // read next group
              }
              else
                  eof = true;
          }
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          if (gathered.ordinality())
          {
              while (gathered.isItem(curIndex))
              {
                  const void * ret = gathered.item(curIndex++);
                  if (stepCompare->docompare(ret, seek, numFields) >= 0)
                  {
                      processed++;
                      return ret;
                  }
                  ReleaseRoxieRow(ret);
              }
              curIndex = 0;
              gathered.kill();
              //nextRowGE never returns an end of group marker.
          }
  
          //Not completely sure about this - it could lead the the start of a group being skipped, 
          //so the group filter could potentially work on a different group.  If so, we'd need to check the
          //next fields were a subset of the grouping fields - more an issue for the group activity.
          
          //MORE: What do we do with wasCompleteMatch?  something like the following????
  #if 0
          for (;;)
          {
              const void * ret;
              if (stepExtra.returnMismatches())
              {
                  bool matchedCompletely = true;
                  ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
                  if (!wasCompleteMatch)
                      return ret;
              }
              else
                  ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  #endif
  
          const void * ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          while (ret)
          {
              gathered.append(ret);
              ret = inputStream->nextRow();
          }
  
          unsigned num = gathered.ordinality();
          if (num != 0)
          {
              if (!helper.isValid(num, (const void * *)gathered.getArray()))
                  ReleaseRoxieRows(gathered);       // read next group
          }
          else
              eof = true;
  
          return IEngineRowStream::ungroupedNextRow();
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          eof = false;
          releaseGathered();
          inputStream->resetEOF();
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
  };
  
  class CRoxieServerFilterGroupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerFilterGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerFilterGroupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerFilterGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerFilterGroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSideEffectActivity : public CRoxieServerActivity
  {
      IHThorSideEffectArg &helper;
      CriticalSection ecrit;
      Owned<IException> exception;
      bool executed;
  public:
  
      CRoxieServerSideEffectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSideEffectArg &)basehelper)
      {
          executed = false;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          CriticalBlock b(ecrit);
          if (exception)
              throw(exception.getLink());
          if (!executed)
          {
              try
              {
                  executed = true;
                  helper.action();
              }
              catch(IException *E)
              {
                  exception.set(E);
                  throw;
              }
          }
          return NULL;
      }
  
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract) 
      {
          CriticalBlock b(ecrit);
          if (exception)
              throw(exception.getLink());
          if (!executed)
          {
              try
              {
                  ActivityTimer t(activityStats, timeActivities);
                  executed = true;
                  start(parentExtractSize, parentExtract, false);
                  helper.action();
                  stop();
              }
              catch(IException *E)
              {
                  ctx->notifyAbort(E);
                  abort();
                  exception.set(E);
                  throw;
              }
          }
      }
  
      virtual void reset()
      {
          executed = false;
          exception.clear();
          CRoxieServerActivity::reset();
      }
  };
  
  class CRoxieServerSideEffectActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerSideEffectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSideEffectActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot && !meta.queryOriginal();
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSideEffectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerSideEffectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerActionActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorActionArg &helper;
  public:
  
      CRoxieServerActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorActionArg &)basehelper)
      {
      }
  
      virtual void onExecute() 
      {
          helper.action();
      }
  };
  
  class CRoxieServerActionActivityFactory : public CRoxieServerInternalSinkFactory
  {
  public:
      CRoxieServerActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerActionActivity(_ctx, this, _probeManager, usageCount);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
  {
      return new CRoxieServerActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerSampleActivity : public CRoxieServerActivity
  {
      IHThorSampleArg &helper;
      unsigned numSamples;
      unsigned numToSkip;
      unsigned whichSample;
      bool anyThisGroup;
      bool eof;
  
  public:
  
      CRoxieServerSampleActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSampleArg &)basehelper)
      {
          numSamples = 0;
          numToSkip = 0;
          whichSample = 0;
          anyThisGroup = false;
          eof = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          anyThisGroup = false;
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numSamples = helper.getProportion();
          whichSample = helper.getSampleNumber();
          numToSkip = (whichSample ? whichSample-1 : 0);
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          for (;;)
          {
              const void * ret = inputStream->nextRow();
              if (!ret)
              {
                  //this does work with groups - may or may not be useful...
                  //reset the sample for each group.... probably best.
                  numToSkip = (whichSample ? whichSample-1 : 0);
                  if (anyThisGroup)
                  {
                      anyThisGroup = false;
                      return NULL;
                  }
                  ret = inputStream->nextRow();
                  if (!ret)
                  {
                      eof = true;
                      return NULL;                // eof...
                  }
              }
  
              if (numToSkip == 0)
              {
                  anyThisGroup = true;
                  numToSkip = numSamples-1;
                  processed++;
                  return ret;
              }
              numToSkip--;
              ReleaseRoxieRow(ret);
          }
      }
  };
  
  class CRoxieServerSampleActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSampleActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSampleActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSampleActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSampleActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChooseSetsActivity : public CRoxieServerActivity
  {
      IHThorChooseSetsArg &helper;
      unsigned numSets;
      unsigned * setCounts;
      bool done;
  
  public:
  
      CRoxieServerChooseSetsActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorChooseSetsArg &)basehelper)
      {
          setCounts = NULL;
          numSets = 0;
          done = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numSets = helper.getNumSets();
          setCounts = new unsigned[numSets];
          memset(setCounts, 0, sizeof(unsigned)*numSets);
          helper.setCounts(setCounts);
      }
  
      virtual void reset()
      {
          delete [] setCounts;
          setCounts = NULL;
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done)
              return NULL;
  
          for (;;)
          {
              const void * ret = inputStream->ungroupedNextRow();
              if (!ret)
              {
                  done = true;
                  return NULL;
              }
              processed++;
              switch (helper.getRecordAction(ret))
              {
              case 2:
                  done = true;
                  return ret;
              case 1:
                  return ret;
              }
  
              ReleaseRoxieRow(ret);
          }
      }
  };
  
  class CRoxieServerChooseSetsActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerChooseSetsActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChooseSetsActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerChooseSetsActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChooseSetsActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerChooseSetsExActivity : public CRoxieServerActivity
  {
  protected:
      IHThorChooseSetsExArg &helper;
      unsigned numSets;
      unsigned curIndex;
      unsigned * setCounts;
      count_t * limits;
      bool done;
      ConstPointerArray gathered;
      virtual bool includeRow(const void * row) = 0;
      virtual void calculateSelection() = 0;
  
  public:
  
      CRoxieServerChooseSetsExActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorChooseSetsExArg &)basehelper)
      {
          setCounts = NULL;
          limits = NULL;
          done = false;
          curIndex = 0;
          numSets = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          curIndex = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numSets = helper.getNumSets();
          setCounts = new unsigned[numSets];
          memset(setCounts, 0, sizeof(unsigned)*numSets);
          limits = (count_t *)calloc(sizeof(count_t), numSets);
          helper.getLimits(limits);
      }
  
      virtual void reset()
      {
          delete [] setCounts;
          setCounts = NULL;
          free(limits);
          limits = NULL;
          roxiemem::ReleaseRoxieRowRange(gathered.getArray(), curIndex, gathered.ordinality());
          gathered.kill();
          curIndex = 0;
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (gathered.ordinality() == 0)
          {
              curIndex = 0;
              if (!inputStream->nextGroup(gathered))
              {
                  done = true;
                  return NULL;
              }
  
              ForEachItemIn(idx1, gathered)
              {
                  unsigned category = helper.getCategory(gathered.item(idx1));
                  if (category)
                      setCounts[category-1]++;
              }
              calculateSelection();
          }
  
          while (gathered.isItem(curIndex))
          {
              const void * row = gathered.item(curIndex);
              gathered.replace(NULL, curIndex);
              curIndex++;
              if (includeRow(row))
              {
                  processed++;
                  return row;
              }
              ReleaseRoxieRow(row);
          }
  
          gathered.kill();
          return NULL;
      }
  };
  
  class CRoxieServerChooseSetsLastActivity : public CRoxieServerChooseSetsExActivity 
  {
      unsigned * numToSkip;
  public:
      CRoxieServerChooseSetsLastActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager) : CRoxieServerChooseSetsExActivity(_ctx, _factory, _probeManager)
      {
          numToSkip = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerChooseSetsExActivity::start(parentExtractSize, parentExtract, paused);
          numToSkip = (unsigned *)calloc(sizeof(unsigned), numSets);
      }
  
      virtual void reset() 
      { 
          free(numToSkip); 
          numToSkip = NULL;
          CRoxieServerChooseSetsExActivity::reset();
      }
  
  protected:
      virtual void calculateSelection()
      {
          for (unsigned idx=0; idx < numSets; idx++)
          {
              if (setCounts[idx] < limits[idx])
                  numToSkip[idx] = 0;
              else
                  numToSkip[idx] = (unsigned)(setCounts[idx] - limits[idx]);
          }
      }
  
      virtual bool includeRow(const void * row)
      {
          unsigned category = helper.getCategory(row);
          if (category)
          {
              if (numToSkip[category-1] == 0)
                  return true;
              numToSkip[category-1]--;
          }
          return false;       
      }
  };
  
  class CRoxieServerChooseSetsEnthActivity : public CRoxieServerChooseSetsExActivity 
  {
      count_t * counter;
  public:
      CRoxieServerChooseSetsEnthActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager) : CRoxieServerChooseSetsExActivity(_ctx, _factory, _probeManager)
      {
          counter = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerChooseSetsExActivity::start(parentExtractSize, parentExtract, paused);
          counter = (count_t *)calloc(sizeof(count_t), numSets);
      }
  
      virtual void reset() 
      { 
          free(counter); 
          counter = NULL;
          CRoxieServerChooseSetsExActivity::reset();
      }
  
  protected:
      virtual void calculateSelection()
      {
      }
  
      virtual bool includeRow(const void * row)
      {
          unsigned category = helper.getCategory(row);
          if (category)
          {
              assertex(category <= numSets);
              counter[category-1] += limits[category-1];
              if(counter[category-1] >= setCounts[category-1])
              {
                  counter[category-1] -= setCounts[category-1];
                  return true;
              }       
          }
          return false;       
      }
  };
  
  class CRoxieServerChooseSetsEnthActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerChooseSetsEnthActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChooseSetsEnthActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerChooseSetsEnthActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChooseSetsEnthActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  class CRoxieServerChooseSetsLastActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerChooseSetsLastActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerChooseSetsLastActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerChooseSetsLastActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerChooseSetsLastActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerEnthActivity : public CRoxieServerActivity
  {
      IHThorEnthArg &helper;
      unsigned __int64 numerator;
      unsigned __int64 denominator;
      unsigned __int64 counter;
      bool eof;
  
  public:
  
      CRoxieServerEnthActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorEnthArg &)basehelper)
      {
          eof = false;
          numerator = denominator = counter = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          numerator = helper.getProportionNumerator();
          denominator = helper.getProportionDenominator();
          if(denominator == 0) denominator = 1; //MORE: simplest way to avoid disaster in this case
          counter = (helper.getSampleNumber()-1) * greatestCommonDivisor(numerator, denominator);
          if (counter >= denominator)
              counter %= denominator;
      }
  
      inline bool wanted()
      {       
          counter += numerator;
          if(counter >= denominator)
          {
              counter -= denominator;
              return true;
          }       
          return false;   
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          const void * ret;
          for (;;)
          {
              ret = inputStream->ungroupedNextRow();
              if (!ret) //eof
              {
                  eof = true;
                  return ret;
              }
              if (wanted())
                  return ret;
              ReleaseRoxieRow(ret);
          }
      }
  };
  
  class CRoxieServerEnthActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerEnthActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerEnthActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerEnthActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerEnthActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerAggregateActivity : public CRoxieServerActivity
  {
      IHThorAggregateArg &helper;
      bool eof;
      bool isInputGrouped;
      bool abortEarly;
  
  public:
      CRoxieServerAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorAggregateArg &)basehelper)
      {
          eof = false;
          isInputGrouped = false;
          abortEarly = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          isInputGrouped = input->queryOutputMeta()->isGrouped();     // could be done earlier, in setInput?
          abortEarly = !isInputGrouped && (factory->getKind() == TAKexistsaggregate); // ditto
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          const void * next = inputStream->nextRow();
          if (!next && isInputGrouped)
          {
              eof = true;
              return NULL;
          }
  
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t finalSize = helper.clearAggregate(rowBuilder);
  
          if (next)
          {
              finalSize = helper.processFirst(rowBuilder, next);
              ReleaseRoxieRow(next);
  
              if (!abortEarly)
              {
                  for (;;)
                  {
                      next = inputStream->nextRow();
                      if (!next)
                          break;
  
                      finalSize = helper.processNext(rowBuilder, next);
                      ReleaseRoxieRow(next);
                  }
              }
          }
  
          if (!isInputGrouped)        // either read all, or aborted early
              eof = true;
  
          processed++;
          return rowBuilder.finalizeRowClear(finalSize);
      }
  };
  
  class CRoxieServerStrandedAggregateActivity : public CRoxieServerStrandedActivity, implements IOrderedCallbackCollection
  {
      class AggregateProcessor : public StrandProcessor, implements IStrandThreaded, implements IOrderedOutputCallback
      {
      protected:
          IHThorAggregateArg &helper;
          OwnedConstRoxieRow result;
          bool eof = false;
          bool isInputGrouped = false;
          bool abortEarly = false;
  
      public:
          AggregateProcessor(CRoxieServerActivity &_parent, IEngineRowStream *_inputStream, IHThorAggregateArg &_helper, bool _isInputGrouped, bool _abortEarly)
          : StrandProcessor(_parent, _inputStream, true), helper(_helper), isInputGrouped(_isInputGrouped), abortEarly(_abortEarly)
          {
          }
          virtual const void * nextRow() override
          {
              ActivityTimer t(activityStats, timeActivities);
              if (eof || isAborting())
                  return NULL;
  
              const void * next = inputStream->nextRow();
              if (!next && isInputGrouped)
              {
                  eof = true;
                  return NULL;
              }
  
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              size32_t finalSize = helper.clearAggregate(rowBuilder);
  
              if (next)
              {
                  finalSize = helper.processFirst(rowBuilder, next);
                  ReleaseRoxieRow(next);
  
                  if (!abortEarly)
                  {
                      for (;;)
                      {
                          next = inputStream->nextRow();
                          if (!next)
                              break;
  
                          finalSize = helper.processNext(rowBuilder, next);
                          ReleaseRoxieRow(next);
                          if (isAborting())
                              break;
                      }
                  }
              }
  
              if (!isInputGrouped)        // either read all, or aborted early
                  eof = true;
  
              processed++;
              return rowBuilder.finalizeRowClear(finalSize);
          }
          virtual void threadmain() override
          {
              isInputGrouped = true; // gather no row if there are no input rows.
              result.setown(nextRow());
              queryActivity().noteStrandFinished(inputStream);
          }
          virtual void reset() override
          {
              StrandProcessor::reset();
              result.clear();
              eof = false;
          }
   //interface IOutputOutputCallback
          virtual bool noteEndOfInputChunk() { return true; }
          virtual void noteEndOfInput() {}
  
          virtual void stopStream()
          {
              inputStream->stop();
          }
          const void * queryResult() const { return result; }
          CRoxieServerStrandedAggregateActivity & queryActivity() { return static_cast<CRoxieServerStrandedAggregateActivity &>(parent); }
      };
  
      //This class is only used when a single aggregate is being calculated.
      class AggregateCombiner : public CInterfaceOf<IEngineRowStream>
      {
      public:
          explicit AggregateCombiner(CRoxieServerStrandedAggregateActivity & _parent) : parent(_parent)
          {
          }
          virtual const void *nextRow()
          {
              return parent.getCombinedAggregate();
          }
          virtual void stop()
          {
              parent.outputStopped();
          }
          virtual void resetEOF()
          {
              throwUnexpected();
          }
      protected:
          CRoxieServerStrandedAggregateActivity & parent;
      } combiner;
  
      class NullCallback : implements IOrderedOutputCallback
      {
      public:
          virtual bool noteEndOfInputChunk() { return true; }
          virtual void noteEndOfInput() {}
      } nullCallback;
  
  public:
      CRoxieServerStrandedAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const StrandOptions &_strandOptions)
          : CRoxieServerStrandedActivity(_ctx, _factory, _probeManager, _strandOptions), combiner(*this), helper((IHThorAggregateArg &)basehelper)
      {
          isInputGrouped = false;
          abortEarly = false;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          CRoxieServerStrandedActivity::setInput(idx, _sourceIdx, _in);
          isInputGrouped = input->queryOutputMeta()->isGrouped();     // could be done earlier, in setInput?
          abortEarly = !isInputGrouped && (factory->getKind() == TAKexistsaggregate); // ditto
  
          //For EXISTS(), and aggregates that rely on the input order, force to execute single threaded
          //MORE: Ordered should be able to process a block at a time and then merge them, similar to disk write processing.
          if (abortEarly || (!isInputGrouped && queryFactory()->isInputOrdered(false, 0)))
              strandOptions.numStrands = 1;
          else if (!isInputGrouped)
              combineStreams = true;
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          PointerArrayOf<IEngineRowStream> localStreams;
          //If streams are being combined, this class should be notified of end of sections, otherwise grouped aggregates can inform a downstream activity.
          IOrderedCallbackCollection * callbacks = combineStreams ? static_cast<IOrderedCallbackCollection *>(this) : orderedCallbacks;
          Owned<IStrandJunction> outJunction = CRoxieServerStrandedActivity::getOutputStreams(ctx, idx, localStreams, consumerOptions, consumerOrdered, callbacks);
          if (localStreams.ordinality() == 1)
              combineStreams = false;  // Select a more efficient/simpler code path
          if (combineStreams)
          {
              assertex(!outJunction); // this needs more thought!
              streams.append(&combiner);
          }
          else
          {
              streams.swapWith(localStreams);
          }
          return outJunction.getClear();
      }
      virtual StrandProcessor *createStrandProcessor(IEngineRowStream *instream)
      {
          if (traceLevel > 4)
              DBGLOG("Create aggregate strand processor %u", strandOptions.numStrands);
          return new AggregateProcessor(*this, instream, (IHThorAggregateArg &) basehelper, isInputGrouped && !combineStreams, abortEarly);
      }
      virtual StrandProcessor *createStrandSourceProcessor(bool inputOrdered) { throwUnexpected(); }
      virtual bool needsAllocator() const { return true; }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) override
      {
          CRoxieServerStrandedActivity::start(parentExtractSize, parentExtract, paused);
          onStartStrands();
          if (combineStreams)
          {
              barrier.setown(createStrandBarrier());
              ForEachItemIn(i, strands)
                  barrier->startStrand(queryAggregate(i));
              active.store(1);
          }
          else
              active.store(0);
      }
      virtual void reset() override
      {
          CRoxieServerStrandedActivity::reset();
          barrier.clear();
          calculated = false;
      }
  
      //Helper functions for the single aggregate case
      const void * getCombinedAggregate()
      {
          if (calculated)
              return NULL;
          calculated= true;
  
          barrier->waitForStrands();
  
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t finalSize = helper.clearAggregate(rowBuilder);
  
          bool isFirst = true;
          for (unsigned i=0; i < numStrands(); i++)
          {
              const void * next = queryAggregate(i).queryResult();
              if (next)
              {
                  if (isFirst)
                  {
                      finalSize = cloneRow(rowBuilder, next, helper.queryOutputMeta());
                      isFirst = false;
                  }
                  else
                      finalSize = helper.mergeAggregate(rowBuilder, next);
              }
          }
  
          return rowBuilder.finalizeRowClear(finalSize);
      }
  
      void outputStopped()
      {
          //only called if generating a single result.
          assertex(combineStreams);
          if (barrier)
          {
              requestAbort(); // if strands are still running, abort them
              barrier->waitForStrands();
          }
          else
          {
              ForEachItemIn(i, strands)
                  queryAggregate(i).stopStream();
          }
          CRoxieServerStrandedActivity::stop();
      }
  
      void noteStrandFinished(IRowStream * stream)
      {
          barrier->noteStrandFinished(stream);
      }
  
      virtual IOrderedOutputCallback * queryCallback(unsigned i)
      {
          //The non-ordered aggregate does not need to process the ordered call back information.
          //If the ordered single aggregate was implemented then it would need to be processed instead of ignored,
          //however it needs some care since the strands are not yet created.  So cannot return strand(i).
          return &nullCallback;
      }
  
      inline AggregateProcessor & queryAggregate(unsigned i)
      {
          return static_cast<AggregateProcessor &>(strands.item(i));
      }
  
  protected:
      IHThorAggregateArg &helper;
      Owned<IStrandBarrier> barrier;
      IEngineRowStream * combinedStream = nullptr;
      bool isInputGrouped;
      bool abortEarly;
      bool combineStreams = false;
      bool calculated = false;
  };
  
  class CRoxieServerAggregateActivityFactory : public CRoxieServerActivityFactory
  {
      StrandOptions strandOptions;
  public:
      CRoxieServerAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), strandOptions(_graphNode)
      {
          Owned<IHThorAggregateArg> helper = (IHThorAggregateArg *) helperFactory();
          if (!(helper->getAggregateFlags() & TAForderedmerge))
              optStableInput = false;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerStrandedAggregateActivity(_ctx, this, _probeManager, strandOptions);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  typedef unsigned t_hashPrefix;
  
  class CRoxieServerHashAggregateActivity : public CRoxieServerActivity
  {
      IHThorHashAggregateArg &helper;
      RowAggregator aggregated;
  
      bool eof;
      bool gathered;
      bool isGroupedAggregate;
  public:
      CRoxieServerHashAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, bool _isGroupedAggregate, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorHashAggregateArg &)basehelper),
            aggregated(helper, helper),
            isGroupedAggregate(_isGroupedAggregate)
      {
          eof = false;
          gathered = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          gathered = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      { 
          aggregated.reset();
          CRoxieServerActivity::reset(); 
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          if (!gathered)
          {
              aggregated.start(rowAllocator, ctx->queryCodeContext(), activityId);
              bool eog = true;
              for (;;)
              {
                  const void * next = inputStream->nextRow();
                  if (!next)
                  {
                      if (isGroupedAggregate)
                      {
                          if (eog)
                              eof = true;
                          break;
                      }
                      next = inputStream->nextRow();
                      if (!next)
                          break;
                  }
                  eog = false;
                  aggregated.addRow(next);
                  ReleaseRoxieRow(next);
              }
              gathered = true;
          }
  
          Owned<AggregateRowBuilder> next = aggregated.nextResult();
          if (next)
          {
              processed++;
              return next->finalizeRowClear();
          }
  
          if (!isGroupedAggregate)
              eof = true;
  
          aggregated.reset();
          gathered = false;
          return NULL;
      }
  };
  
  class CRoxieServerHashAggregateActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerHashAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          isGroupedAggregate = _graphNode.getPropBool("att[@name='grouped']/@value");
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerHashAggregateActivity(_ctx, this, isGroupedAggregate, _probeManager);
      }
  protected:
      bool isGroupedAggregate;
  };
  
  IRoxieServerActivityFactory *createRoxieServerHashAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerHashAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerDegroupActivity : public CRoxieServerActivity
  {
      bool eof;
  
  public:
      CRoxieServerDegroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          eof = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          const void * ret = inputStream->ungroupedNextRow();
          if (ret)
              processed++;
          else
              eof = true;
          return ret;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          const void * ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (ret)
              processed++;
          else
              eof = true;
          return ret;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          eof = false;
          inputStream->resetEOF();
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
  };
  
  class CRoxieServerDegroupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerDegroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerDegroupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDegroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerDegroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSpillReadActivity : public CRoxieServerActivity
  {
      IHThorDiskReadArg &helper;
      bool needTransform;
      bool eof;
      bool anyThisGroup;
      unsigned __int64 rowLimit;
      unsigned __int64 choosenLimit;
  
  public:
      CRoxieServerSpillReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorDiskReadArg &)basehelper)
      {
          needTransform = helper.needTransform();
          rowLimit = (unsigned __int64) -1;
          choosenLimit = 0;
          eof = false;
          anyThisGroup = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          anyThisGroup = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (helper.canMatchAny())
              eof = false;
          else
              eof = true;
          choosenLimit = helper.getChooseNLimit();
          rowLimit = helper.getRowLimit();
          helper.setCallback(NULL);       // members should not be called - change if they are
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          if (processed==choosenLimit)
          {
              eof = true;
              return NULL;
          }
          if (needTransform)
          {
              for (;;)
              {
                  OwnedConstRoxieRow in = inputStream->nextRow();
                  if (!in)
                  {
                      if (anyThisGroup)
                      {
                          anyThisGroup = false;
                          return NULL;
                      }
                      in.setown(inputStream->nextRow());
                      if (!in)
                      {
                          eof = true;
                          return NULL;                // eof...
                      }
                  }
                  if (likely(helper.canMatch(in)))
                  {
                      unsigned outSize;
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      try
                      {
                          outSize = helper.transform(rowBuilder, in);
                      }
                      catch (IException *E)
                      {
                          throw makeWrappedException(E);
                      }
  
                      if (outSize)
                      {
                          anyThisGroup = true;
                          processed++;
                          if (processed==rowLimit)
                          {
                              if (traceLevel > 4)
                                  DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                              helper.onLimitExceeded();
                          }
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                  }
              }
          }
          else
          {
              const void *ret = inputStream->nextRow();
              if (ret)
              {
                  processed++;
                  if (processed==rowLimit)
                  {
                      if (traceLevel > 4)
                          DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                      ReleaseClearRoxieRow(ret);
                      helper.onLimitExceeded(); // should not return
                      throwUnexpected();
                  }
              }
              return ret;
          }
      }
  };
  
  class CRoxieServerSpillReadActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSpillReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSpillReadActivity(_ctx, this, _probeManager);
      }
  
      virtual void addDependency(unsigned source, ThorActivityKind sourceKind, unsigned sourceIdx, int controlId, const char *edgeId)
      {
          if (sourceKind==TAKspill || sourceKind==TAKdiskwrite || sourceKind==TAKspillwrite) // Bit of a hack - codegen probably should differentiate
              setInput(0, source, sourceIdx);
          else
              CRoxieServerActivityFactory::addDependency(source, kind, sourceIdx, controlId, edgeId);
      } 
  };
  
  IRoxieServerActivityFactory *createRoxieServerSpillReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSpillReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSpillWriteActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerSpillWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
      }
  
      ~CRoxieServerSpillWriteActivity()
      {
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          return inputStream->nextRow();
      }
  };
  
  //==================================================================================
  
  class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity, implements IRoxiePublishCallback
  {
  protected:
      Owned<IExtRowWriter> outSeq;
      Owned<IOutputRowSerializer> rowSerializer;
      Linked<IFileIOStream> diskout;
      bool blockcompressed;
      bool extend;
      bool overwrite;
      bool encrypted;
      bool grouped;
      IHThorDiskWriteArg &helper;
      Owned<IRecordSize> diskmeta;
      Owned<IRoxieWriteHandler> writer;
  
      bool tallycrc;
      unsigned __int64 uncompressedBytesWritten;
      CRC32 crc;
  
      void updateWorkUnitResult(unsigned __int64 reccount)
      {
          assertex(writer);
          // MORE - a lot of this is common with hthor
          WorkunitUpdate wu = ctx->updateWorkUnit();
          if (wu)
          {
              OwnedRoxieString rawLogicalName = helper.getFileName();
              StringBuffer lfn;   // logical filename
              expandLogicalFilename(lfn, rawLogicalName, wu, false, false);
              if (lfn.length())
              {
                  unsigned flags = helper.getFlags();
                  WUFileKind fileKind;
                  if (TDXtemporary & flags)
                      fileKind = WUFileTemporary;
                  else if(TDXjobtemp & flags)
                      fileKind = WUFileJobOwned;
                  else if(TDWowned & flags)
                      fileKind = WUFileOwned;
                  else
                      fileKind = WUFileStandard;
                  StringArray clusters;
                  writer->getClusters(clusters);
                  wu->addFile(lfn.str(), &clusters, helper.getTempUsageCount(), fileKind, NULL);
                  if (!(flags & TDXtemporary) && helper.getSequence() >= 0)
                  {
                      Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
                      if (result)
                      {
                          result->setResultTotalRowCount(reccount); 
                          result->setResultStatus(ResultStatusCalculated);
                          if (helper.getFlags() & TDWresult)
                              result->setResultFilename(lfn.str());
                          else
                              result->setResultLogicalName(lfn.str());
                      }
                  }
              }
          }
      }
  
      void resolve()
      {
          OwnedRoxieString rawLogicalName = helper.getFileName();
          assertex(rawLogicalName);
          assertex((helper.getFlags() & TDXtemporary) == 0);
          StringArray clusters;
          unsigned clusterIdx = 0;
          while(true)
          {
              OwnedRoxieString cluster(helper.getCluster(clusterIdx));
              if(!cluster)
                  break;
              clusters.append(cluster);
              clusterIdx++;
          }
          if (clusters.length())
          {
              if (extend)
                  throw MakeStringException(0, "Cannot combine EXTEND and CLUSTER flags on disk write of file %s", rawLogicalName.get());
          }
          else
          {
              StringBuffer defaultCluster;
              if (getDefaultStoragePlane(defaultCluster))
                  clusters.append(defaultCluster);
              else if (roxieName.length())
                  clusters.append(roxieName.str());
              else
                  clusters.append(".");
          }
          // Using defaultPrivilegedUser as restricted files applies to limits reading of file (not writing of files)
          writer.setown(ctx->createLFN(rawLogicalName, overwrite, extend, clusters, defaultPrivilegedUser));
          // MORE - need to check somewhere that single part if it's an existing file or an external one...
      }
  
  public:
      CRoxieServerDiskWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0), helper((IHThorDiskWriteArg &) basehelper)
      {
          extend = ((helper.getFlags() & TDWextend) != 0);
          overwrite = ((helper.getFlags() & TDWoverwrite) != 0);
          grouped = (helper.getFlags() & TDXgrouped) != 0;
          diskmeta.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
          if (grouped)
              diskmeta.setown(createDeltaRecordSize(diskmeta, +1));
          size32_t fixedSize = diskmeta->getFixedSize();
          blockcompressed = (((helper.getFlags() & TDWnewcompress) != 0) || (((helper.getFlags() & TDXcompress) != 0) && ((0 == fixedSize) || (fixedSize >= MIN_ROWCOMPRESS_RECSIZE)))); //always use new compression
          encrypted = false; // set later
          tallycrc = true;
          uncompressedBytesWritten = 0;
      }
  
      ~CRoxieServerDiskWriteActivity()
      {
      }
  
      virtual bool needsAllocator() const
      {
          return true;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          
          resolve();
          Owned<IFileIO> io;
          void *ekey;
          size32_t ekeylen;
          helper.getEncryptKey(ekeylen, ekey);
          Owned<ICompressor> ecomp;
          if (ekeylen!=0) 
          {
              ecomp.setown(createAESCompressor256(ekeylen,ekey));
              memset(ekey,0,ekeylen);
              rtlFree(ekey);
              encrypted = true;
              blockcompressed = true;
          }
          if (blockcompressed)
              io.setown(createCompressedFileWriter(writer->queryFile(), (diskmeta->isFixedSize() ? diskmeta->getFixedSize() : 0), extend, true, ecomp, COMPRESS_METHOD_LZW));
          else
              io.setown(writer->queryFile()->open(extend ? IFOwrite : IFOcreate));
          if (!io)
              throw MakeStringException(errno, "Failed to create%s file %s for writing", (encrypted ? " encrypted" : (blockcompressed ? " compressed" : "")), writer->queryFile()->queryFilename());
          diskout.setown(createBufferedIOStream(io));
          if (extend)
              diskout->seek(0, IFSend);
          tallycrc = !blockcompressed; // MORE: Should this be controlled by an activity hint/flag?
          Owned<IRowInterfaces> rowIf = createRowInterfaces(input->queryOutputMeta(), activityId, factory->getHeapFlags(), ctx->queryCodeContext());
          rowSerializer.set(rowIf->queryRowSerializer());
          unsigned rwFlags = rw_autoflush;
          if(grouped)
              rwFlags |= rw_grouped;
          if(tallycrc)
              rwFlags |= rw_crc;
          outSeq.setown(createRowWriter(diskout, rowIf, rwFlags));
      }
  
      virtual void stop()
      {
          if (aborted)
          {
              if (writer)
                  writer->finish(false, this);
          }
          else
          {
              if (outSeq)
                  outSeq->flush(&crc);
              if (outSeq)
                  uncompressedBytesWritten = outSeq->getPosition();
              outSeq.clear();
              diskout.clear();  // Make sure file is properly closed or date may not match published info
              if (writer)
              {
                  updateWorkUnitResult(processed);
                  writer->finish(true, this);
              }
          }
          writer.clear();
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          CRoxieServerInternalSinkActivity::reset();
          diskout.clear();
          outSeq.clear();
          writer.clear();
          uncompressedBytesWritten = 0;
          crc.reset();
      }
  
      virtual void onExecute()
      {
          for (;;)
          {
              const void *row = inputStream->nextRow();
              if (!row)
              {
                  row = inputStream->nextRow();
                  if (!row)
                      break;
                  if (grouped)
                      outSeq->putRow(NULL);
              }
              processed++;
              outSeq->putRow(row);
          }
      }
  
      virtual void setFileProperties(IFileDescriptor *desc) const
      {
          IPropertyTree &partProps = desc->queryPart(0)->queryProperties(); //properties of the first file part.
          IPropertyTree &fileProps = desc->queryProperties(); // properties of the logical file
          if (blockcompressed)
          {
              // caller has already set @size from file size...
              fileProps.setPropBool("@blockCompressed", true);
              partProps.setPropInt64("@compressedSize", partProps.getPropInt64("@size", 0));
              partProps.setPropInt64("@size", uncompressedBytesWritten);
          }
          else if (tallycrc && crc.get())
              partProps.setPropInt64("@fileCrc", crc.get());
  
          if (encrypted)
              fileProps.setPropBool("@encrypted", true);
  
          fileProps.setPropInt64("@recordCount", processed);
          partProps.setPropInt64("@recordCount", processed);
  
          unsigned flags = helper.getFlags();
          if (flags & TDWpersist)
              fileProps.setPropBool("@persistent", true);
          if (grouped)
              fileProps.setPropBool("@grouped", true);
          if (flags & (TDWowned|TDXjobtemp|TDXtemporary))
              fileProps.setPropBool("@owned", true);
          if (flags & TDWresult)
              fileProps.setPropBool("@result", true);
  
          IConstWorkUnit *workUnit = ctx->queryWorkUnit();
          if (workUnit)
          {
              fileProps.setProp("@owner", workUnit->queryUser());
              fileProps.setProp("@workunit", workUnit->queryWuid());
              fileProps.setProp("@job", workUnit->queryJobName());
          }
          if (flags & TDWexpires)
              setExpiryTime(fileProps, helper.getExpiryDays());
          if (flags & TDWupdate)
          {
              unsigned eclCRC;
              unsigned __int64 totalCRC;
              helper.getUpdateCRCs(eclCRC, totalCRC);
              fileProps.setPropInt("@eclCRC", eclCRC);
              fileProps.setPropInt64("@totalCRC", totalCRC);
          }
          fileProps.setPropInt("@formatCrc", helper.getFormatCrc());
          if (flags & TDWrestricted)
              fileProps.setPropBool("restricted", true);
          IRecordSize * inputMeta = input->queryOutputMeta();
          if ((inputMeta->isFixedSize()) && !isOutputTransformed())
              fileProps.setPropInt("@recordSize", inputMeta->getFixedSize() + (grouped ? 1 : 0));
  
          const char *recordECL = helper.queryRecordECL();
          if (recordECL && *recordECL)
              fileProps.setProp("ECL", recordECL);
          setRtlFormat(fileProps, helper.queryDiskRecordSize());
  
          fileProps.setProp("@kind", "flat"); // default, derivitives may override
      }
  
      virtual IUserDescriptor *queryUserDescriptor() const
      {
          IConstWorkUnit *workUnit = ctx->queryWorkUnit();
          if (workUnit)
              return workUnit->queryUserDescriptor();//ad-hoc mode
          else
          {
              Owned<IRoxieDaliHelper> daliHelper = connectToDali(false);
              if (daliHelper)
                  return daliHelper->queryUserDescriptor();//predeployed query mode
          }
          return NULL;
      }
  
      virtual bool isOutputTransformed() const { return false; }
  
  };
  
  //=================================================================================
  
  class CRoxieServerCsvWriteActivity : public CRoxieServerDiskWriteActivity
  {
      IHThorCsvWriteArg &csvHelper;
      CSVOutputStream csvOutput;
  
  public:
      CRoxieServerCsvWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerDiskWriteActivity(_ctx, _factory, _probeManager), csvHelper(static_cast<IHThorCsvWriteArg &>(helper))
      {
          csvOutput.init(csvHelper.queryCsvParameters(), false);
      }
  
      virtual void onExecute() 
      {
          OwnedRoxieString header(csvHelper.queryCsvParameters()->getHeader());
          if (header) 
          {
              csvOutput.beginLine();
              csvOutput.writeHeaderLn(strlen(header), header);
              diskout->write(csvOutput.length(), csvOutput.str());
          }
          for (;;)
          {
              const void *nextrec = inputStream->ungroupedNextRow();
              if (!nextrec)
                  break;
              processed++;
              csvOutput.beginLine();
              csvHelper.writeRow((const byte *)nextrec, &csvOutput);
              csvOutput.endLine();
              diskout->write(csvOutput.length(), csvOutput.str());
              ReleaseRoxieRow(nextrec);
          }
          OwnedRoxieString footer(csvHelper.queryCsvParameters()->getFooter());
          if (footer) 
          {
              csvOutput.beginLine();
              csvOutput.writeHeaderLn(strlen(footer), footer);
              diskout->write(csvOutput.length(), csvOutput.str());
          }
      }
  
      virtual void setFileProperties(IFileDescriptor *desc) const
      {
          CRoxieServerDiskWriteActivity::setFileProperties(desc);
          IPropertyTree &props = desc->queryProperties();
          props.setProp("@format","utf8n");
  
          ICsvParameters *csvParameters = csvHelper.queryCsvParameters();
          StringBuffer separator;
          OwnedRoxieString rs(csvParameters->getSeparator(0));
          const char *s = rs;
          while (s &&  *s)
          {
              if (',' == *s)
                  separator.append("\\,");
              else
                  separator.append(*s);
              ++s;
          }
          props.setProp("@csvSeparate", separator.str());
          props.setProp("@csvQuote", rs.setown(csvParameters->getQuote(0)));
          props.setProp("@csvTerminate", rs.setown(csvParameters->getTerminator(0)));
          props.setProp("@csvEscape", rs.setown(csvParameters->getEscape(0)));
          props.setProp("@kind", "csv");
      }
  
      virtual bool isOutputTransformed() const { return true; }
  };
  
  
  class CRoxieServerXmlWriteActivity : public CRoxieServerDiskWriteActivity
  {
      IHThorXmlWriteArg &xmlHelper;
      StringAttr rowTag;
      ThorActivityKind kind;
      unsigned headerLength;
      unsigned footerLength;
  
  public:
      CRoxieServerXmlWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, ThorActivityKind _kind)
          : CRoxieServerDiskWriteActivity(_ctx, _factory, _probeManager), xmlHelper(static_cast<IHThorXmlWriteArg &>(helper)), kind(_kind), headerLength(0), footerLength(0)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerDiskWriteActivity::start(parentExtractSize, parentExtract, paused);
          OwnedRoxieString xmlpath(xmlHelper.getXmlIteratorPath());
          if (!xmlpath)
              rowTag.set(DEFAULTXMLROWTAG);
          else
          {
              const char *path = xmlpath;
              if (*path == '/') path++;
              if (strchr(path, '/')) UNIMPLEMENTED;               // more what do we do with /mydata/row
              rowTag.set(path);
          }
      }
  
      virtual void onExecute() 
      {
          StringBuffer header;
          OwnedRoxieString suppliedHeader(xmlHelper.getHeader());
          if (kind==TAKjsonwrite)
              buildJsonHeader(header, suppliedHeader, rowTag);
          else if (suppliedHeader)
              header.set(suppliedHeader);
          else
              header.append(DEFAULTXMLHEADER).newline();
  
          headerLength = header.length();
          diskout->write(header.length(), header.str());
  
          Owned<IXmlWriterExt> writer = createIXmlWriterExt(xmlHelper.getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSONRootless : WTStandard);
          writer->outputBeginArray(rowTag); //need to set this
          writer->clear(); //but not output it
  
          for (;;)
          {
              OwnedConstRoxieRow nextrec = inputStream->ungroupedNextRow();
              if (!nextrec)
                  break;
              processed++;
              writer->clear().outputBeginNested(rowTag, false);
              xmlHelper.toXML((const byte *)nextrec.get(), *writer);
              writer->outputEndNested(rowTag);
              diskout->write(writer->length(), writer->str());
          }
          OwnedRoxieString suppliedFooter(xmlHelper.getFooter());
          StringBuffer footer;
          if (kind==TAKjsonwrite)
              buildJsonFooter(footer.newline(), suppliedFooter, rowTag);
          else if (suppliedFooter)
              footer.append(suppliedFooter);
          else
              footer.append(DEFAULTXMLFOOTER);
  
          footerLength=footer.length();
          diskout->write(footer.length(), footer);
      }
  
      virtual void reset()
      {
          CRoxieServerDiskWriteActivity::reset();
          rowTag.clear();
      }
  
      virtual void setFileProperties(IFileDescriptor *desc) const
      {
          CRoxieServerDiskWriteActivity::setFileProperties(desc);
          desc->queryProperties().setProp("@format","utf8n");
          desc->queryProperties().setProp("@rowTag",rowTag.get());
          desc->queryProperties().setProp("@kind", (kind==TAKjsonwrite) ? "json" : "xml");
  
          desc->queryProperties().setPropInt(FPheaderLength, headerLength);
          desc->queryProperties().setPropInt(FPfooterLength, footerLength);
      }
  
      virtual bool isOutputTransformed() const { return true; }
  };
  
  class CRoxieServerDiskWriteActivityFactory : public CRoxieServerMultiOutputFactory
  {
      bool isTemp;
  public:
      CRoxieServerDiskWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiOutputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorDiskWriteArg> helper = (IHThorDiskWriteArg *) helperFactory();
          isTemp = (helper->getFlags() & TDXtemporary) != 0;
          setNumOutputs(helper->getTempUsageCount());
          if (_kind != TAKspillwrite)
              assertex(numOutputs == 0);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          switch (numOutputs)
          {
          case 0:
              switch (kind)
              {
              case TAKdiskwrite:
                  return new CRoxieServerDiskWriteActivity(_ctx, this, _probeManager);
              case TAKcsvwrite:
                  return new CRoxieServerCsvWriteActivity(_ctx, this, _probeManager);
              case TAKxmlwrite:
              case TAKjsonwrite:
                  return new CRoxieServerXmlWriteActivity(_ctx, this, _probeManager, kind);
              case TAKspillwrite:
                  return new CRoxieServerNullSinkActivity(_ctx, this, _probeManager);
              };
              throwUnexpected();
  
          case 1:
              return new CRoxieServerSpillWriteActivity(_ctx, this, _probeManager);
          default:
              return new CRoxieServerThroughSpillActivity(_ctx, this, _probeManager, numOutputs);
          }
      }
  
      virtual bool isSink() const
      {
          return numOutputs == 0 && (kind==TAKspillwrite || !isTemp); // MORE - check with Gavin if this is right if not a temp but reread in  same job...
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerDiskWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerDiskWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity, implements IRoxiePublishCallback
  {
      IHThorIndexWriteArg &helper;
      bool overwrite;
      Owned<ClusterWriteHandler> clusterHandler;
      Owned<IRoxieWriteHandler> writer;
      unsigned __int64 reccount;
      unsigned int fileCrc;
      StringBuffer filename;
      unsigned __int64 duplicateKeyCount = 0;
      unsigned __int64 cummulativeDuplicateKeyCount = 0;
  
      void updateWorkUnitResult()
      {
          // MORE - a lot of this is common with hthor
          WorkunitUpdate wu = ctx->updateWorkUnit();
          if (wu)
          {
              OwnedRoxieString rawLogicalName = helper.getFileName();
              StringBuffer lfn;   // logical filename
              expandLogicalFilename(lfn, rawLogicalName, wu, false, false);
              if (lfn.length())
              {
                  if (helper.getSequence() >= 0)
                  {
                      Owned<IWUResult> result = wu->updateResultBySequence(helper.getSequence());
                      if (result)
                      {
                          result->setResultTotalRowCount(reccount);
                          result->setResultStatus(ResultStatusCalculated);
                          result->setResultLogicalName(lfn.str());
                      }
                  }
                  if(clusterHandler)
                      clusterHandler->finish(writer->queryFile());
                  CTXLOG("Created roxie index file %s", lfn.str());
              }
          }
      }
  
      virtual void resolve()
      {
          StringArray clusters;
          unsigned clusterIdx = 0;
          while(true)
          {
              OwnedRoxieString cluster(helper.getCluster(clusterIdx));
              if(!cluster)
                  break;
              clusters.append(cluster);
              clusterIdx++;
          }
  
          if (!clusters.length())
          {
              StringBuffer defaultCluster;
              if (getDefaultStoragePlane(defaultCluster))
                  clusters.append(defaultCluster);
              else if (roxieName.length())
                  clusters.append(roxieName.str());
              else
                  clusters.append(".");
          }
  
          OwnedRoxieString fname(helper.getFileName());
          writer.setown(ctx->createLFN(fname, overwrite, false, clusters, defaultPrivilegedUser)); // MORE - if there's a workunit, use if for scope.
          filename.set(writer->queryFile()->queryFilename());
          if (writer->queryFile()->exists())
          {
              if (overwrite)
              {
                  CTXLOG("Removing existing %s from DFS",filename.str());
                  writer->queryFile()->remove();
              }
              else
                  throw MakeStringException(99, "Cannot write index file %s, file already exists (missing OVERWRITE attribute?)", filename.str());
          }
      }
  
      void buildUserMetadata(Owned<IPropertyTree> & metadata)
      {
          size32_t nameLen;
          char * nameBuff;
          size32_t valueLen;
          char * valueBuff;
          unsigned idx = 0;
          while(helper.getIndexMeta(nameLen, nameBuff, valueLen, valueBuff, idx++))
          {
              StringBuffer name(nameLen, nameBuff);
              StringBuffer value(valueLen, valueBuff);
              rtlFree(nameBuff);
              rtlFree(valueBuff);
              if(*name == '_' && !checkReservedMetadataName(name))
              {
                  OwnedRoxieString fname(helper.getFileName());
                  throw MakeStringException(0, "Invalid name %s in user metadata for index %s (names beginning with underscore are reserved)", name.str(), fname.get());
              }
              if(!validateXMLTag(name.str()))
              {
                  OwnedRoxieString fname(helper.getFileName());
                  throw MakeStringException(0, "Invalid name %s in user metadata for index %s (not legal XML element name)", name.str(), fname.get());
              }
              if(!metadata)
                  metadata.setown(createPTree("metadata", ipt_fast));
              metadata->setProp(name.str(), value.str());
          }
      }
  
      void buildLayoutMetadata(Owned<IPropertyTree> & metadata)
      {
          if(!metadata)
              metadata.setown(createPTree("metadata", ipt_fast));
          metadata->setProp("_record_ECL", helper.queryRecordECL());
  
          setRtlFormat(*metadata, helper.queryDiskRecordSize());
      }
  
  public:
      CRoxieServerIndexWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, 0), helper(static_cast<IHThorIndexWriteArg &>(basehelper))
      {
          overwrite = ((helper.getFlags() & TIWoverwrite) != 0);
          reccount = 0;
          fileCrc = 0;
      }
  
      ~CRoxieServerIndexWriteActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          resolve();
      }
  
      virtual void onExecute()
      {
          bool isVariable = helper.queryDiskRecordSize()->isVariableSize();
          size32_t fileposSize = hasTrailingFileposition(helper.queryDiskRecordSize()->queryTypeInfo()) ? sizeof(offset_t) : 0;
          size32_t maxDiskRecordSize;
          if (isVariable)
          {
              if (helper.getFlags() & TIWmaxlength)
                  maxDiskRecordSize = helper.getMaxKeySize();
              else
                  maxDiskRecordSize = KEYBUILD_MAXLENGTH; // Current default behaviour, could be improved in the future
          }
          else
              maxDiskRecordSize = helper.queryDiskRecordSize()->getFixedSize()-fileposSize;
  
          if (maxDiskRecordSize > KEYBUILD_MAXLENGTH)
              throw MakeStringException(99, "Index maximum record length (%d) exceeds 32k internal limit", maxDiskRecordSize);
  
          OwnedMalloc<char> rowBuffer(maxDiskRecordSize, true);
  
          fileCrc = -1;
          {
              Owned<IFileIO> io;
              try
              {
                  io.setown(writer->queryFile()->open(IFOcreate));
              }
              catch(IException * e)
              {
                  e->Release();
                  clearKeyStoreCache(false);
                  io.setown(writer->queryFile()->open(IFOcreate));
              }
              if(!io)
                  throw MakeStringException(errno, "Failed to create file %s for writing", filename.str());
  
              bool needsSeek = true;
              unsigned flags = COL_PREFIX | HTREE_FULLSORT_KEY;
              if (helper.getFlags() & TIWrowcompress)
                  flags |= HTREE_COMPRESSED_KEY|HTREE_QUICK_COMPRESSED_KEY;
              else if (!(helper.getFlags() & TIWnolzwcompress))
                  flags |= HTREE_COMPRESSED_KEY;
              if (isVariable)
                  flags |= HTREE_VARSIZE;
              Owned<IPropertyTree> metadata;
              buildUserMetadata(metadata);
              buildLayoutMetadata(metadata);
              unsigned nodeSize = metadata->getPropInt("_nodeSize", NODESIZE);
              if (metadata->getPropBool("_noSeek", ctx->queryOptions().noSeekBuildIndex))
              {
                  flags |= TRAILING_HEADER_ONLY;
                  needsSeek = false;
              }
              if (metadata->getPropBool("_useTrailingHeader", true))
                  flags |= USE_TRAILING_HEADER;
              Owned<IFileIOStream> out = createBufferedIOStream(io, 0x100000);
              if (!needsSeek)
                  out.setown(createNoSeekIOStream(out));
  
              Owned<IKeyBuilder> builder = createKeyBuilder(out, flags, maxDiskRecordSize, nodeSize, helper.getKeyedSize(), 0, &helper, true, false);
              class BcWrapper : implements IBlobCreator
              {
                  IKeyBuilder *builder;
              public:
                  BcWrapper(IKeyBuilder *_builder) : builder(_builder) {}
                  virtual unsigned __int64 createBlob(size32_t size, const void * ptr)
                  {
                      return builder->createBlob(size, (const char *) ptr);
                  }
              } bc(builder);
  
              // Loop thru the results
              for (;;)
              {
                  OwnedConstRoxieRow nextrec(inputStream->ungroupedNextRow());
                  if (!nextrec)
                      break;
                  try
                  {
                      unsigned __int64 fpos=0;
                      RtlStaticRowBuilder rowBuilder(rowBuffer, maxDiskRecordSize);
                      size32_t thisSize = helper.transform(rowBuilder, nextrec, &bc, fpos);
                      builder->processKeyData(rowBuffer, fpos, thisSize);
                  }
                  catch(IException * e)
                  {
                      throw makeWrappedException(e);
                  }
                  reccount++;
              }
              duplicateKeyCount = builder->getDuplicateCount();
              cummulativeDuplicateKeyCount += duplicateKeyCount;
              builder->finish(metadata, &fileCrc);
              clearKeyStoreCache(false);
          }
      }
  
      virtual void stop()
      {
          if (writer)
          {
              if (!aborted)
                  updateWorkUnitResult();
              writer->finish(!aborted, this);
              writer.clear();
          }
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          noteStatistic(StNumDuplicateKeys, cummulativeDuplicateKeyCount);
          CRoxieServerActivity::reset();
          writer.clear();
      }
  
      //interface IRoxiePublishCallback
  
      virtual void setFileProperties(IFileDescriptor *desc) const
      {
          // Now publish to name services
          StringBuffer dir,base;
          offset_t indexFileSize = writer->queryFile()->size();
          if(clusterHandler)
              clusterHandler->splitPhysicalFilename(dir, base);
          else
          {
              // Check filename is URL and get localpath, only actually necessary if force remote reads are are on
              RemoteFilename rfn;
              rfn.setRemotePath(filename);
              StringBuffer localPath;
              rfn.getLocalPath(localPath);
              splitFilename(localPath, &dir, &dir, &base, &base);
          }
  
          desc->setDefaultDir(dir.str());
  
          //properties of the first file part.
          Owned<IPropertyTree> attrs;
          if(clusterHandler)
              attrs.setown(createPTree("Part", ipt_fast));  // clusterHandler is going to set attributes
          else
          {
              // add cluster
              StringBuffer mygroupname;
              desc->setNumParts(1);
              desc->setPartMask(base.str());
              attrs.set(&desc->queryPart(0)->queryProperties());
          }
          attrs->setPropInt64("@size", indexFileSize);
          attrs->setPropInt64("@recordCount", reccount);
  
          CDateTime createTime, modifiedTime, accessedTime;
          writer->queryFile()->getTime(&createTime, &modifiedTime, &accessedTime);
          // round file time down to nearest sec. Nanosec accurancy is not preserved elsewhere and can lead to mismatch later.
          unsigned hour, min, sec, nanosec;
          modifiedTime.getTime(hour, min, sec, nanosec);
          modifiedTime.setTime(hour, min, sec, 0);
          StringBuffer timestr;
          modifiedTime.getString(timestr);
          if(timestr.length())
              attrs->setProp("@modified", timestr.str());
  
          if(clusterHandler)
              clusterHandler->setDescriptorParts(desc, base.str(), attrs);
  
          // properties of the logical file
          IPropertyTree & properties = desc->queryProperties();
          properties.setProp("@kind", "key");
          properties.setPropInt64("@size", indexFileSize);
          properties.setPropInt64("@recordCount", reccount);
          properties.setPropInt64("@duplicateKeyCount", duplicateKeyCount);
          WorkunitUpdate workUnit = ctx->updateWorkUnit();
          if (workUnit)
          {
              properties.setProp("@owner", workUnit->queryUser());
              properties.setProp("@workunit", workUnit->queryWuid());
              properties.setProp("@job", workUnit->queryJobName());
          }
          char const * rececl = helper.queryRecordECL();
          if(rececl && *rececl)
              properties.setProp("ECL", rececl);
  
          if (helper.getFlags() & TIWexpires)
              setExpiryTime(properties, helper.getExpiryDays());
          if (helper.getFlags() & TIWupdate)
          {
              unsigned eclCRC;
              unsigned __int64 totalCRC;
              helper.getUpdateCRCs(eclCRC, totalCRC);
              properties.setPropInt("@eclCRC", eclCRC);
              properties.setPropInt64("@totalCRC", totalCRC);
          }
  
          properties.setPropInt("@fileCrc", fileCrc);
          properties.setPropInt("@formatCrc", helper.getFormatCrc());
          // Legacy record layout info
          void * layoutMetaBuff;
          size32_t layoutMetaSize;
          if(helper.getIndexLayout(layoutMetaSize, layoutMetaBuff))
          {
              properties.setPropBin("_record_layout", layoutMetaSize, layoutMetaBuff);
              rtlFree(layoutMetaBuff);
          }
          if (helper.getFlags() & TIWrestricted)
              properties.setPropBool("restricted", true);
          // New record layout info
          setRtlFormat(properties, helper.queryDiskRecordSize());
          // Bloom info
          const IBloomBuilderInfo * const *bloomFilters = helper.queryBloomInfo();
          while (bloomFilters && *bloomFilters)
          {
              const IBloomBuilderInfo *info = *bloomFilters++;
              IPropertyTree *bloom = properties.addPropTree("Bloom");
              bloom->setPropInt64("@bloomFieldMask", info->getBloomFields());
              bloom->setPropInt64("@bloomLimit", info->getBloomLimit());  // MORE - if we didn't actually build because of the limit that might be interesting. Though that's going to vary by part.
              VStringBuffer pval("%f", info->getBloomProbability());
              bloom->setProp("@bloomProbability", pval.str());
          }
      }
  
      IUserDescriptor *queryUserDescriptor() const
      {
          IConstWorkUnit *workUnit = ctx->queryWorkUnit();
          if (workUnit)
              return workUnit->queryUserDescriptor();//ad-hoc mode
          else
          {
              Owned<IRoxieDaliHelper> daliHelper = connectToDali(false);
              if (daliHelper)
                  return daliHelper->queryUserDescriptor();//predeployed query mode
          }
          return NULL;
      }
  };
  
  //=================================================================================
  
  class CRoxieServerIndexWriteActivityFactory : public CRoxieServerMultiOutputFactory
  {
  public:
      CRoxieServerIndexWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerMultiOutputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, indexWriteStatistics)
      {
          setNumOutputs(0);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerIndexWriteActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return true;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerIndexWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  static inline void getLimitType(unsigned flags, bool & limitFail, bool & limitOnFail) 
  { 
      if((flags & JFmatchAbortLimitSkips) != 0) 
      { 
          limitFail = false; 
          limitOnFail = false; 
      } 
      else 
      { 
          limitOnFail = ((flags & JFonfail) != 0); 
          limitFail = !limitOnFail; 
      } 
  }
  
  class CRoxieServerJoinActivity : public CRoxieServerTwoInputActivity
  {
      enum { JSfill, JSfillleft, JSfillright, JScollate, JScompare, JSleftonly, JSrightonly } state;
  
      IHThorJoinArg &helper;
      ICompare * collate;
      ICompare * collateupper;
  
      ThorActivityKind activityKind;
      bool leftOuterJoin;
      bool rightOuterJoin;
      bool exclude;
      bool limitFail;
      bool limitOnFail;
      bool forceSpill;
      unsigned keepLimit;
      unsigned joinLimit;
      unsigned atmostLimit;
      unsigned abortLimit;
      unsigned atmostsTriggered;
      bool betweenjoin;
  
      OwnedRowArray right;
      const void * left;
      const void * pendingRight;
      unsigned rightIndex;
      unsigned joinCounter;
      BoolArray matchedRight;
      bool matchedLeft;
      Owned<IException> failingLimit;
      ConstPointerArray filteredRight;
      Owned<IRHLimitedCompareHelper> limitedhelper;
  
      OwnedConstRoxieRow defaultLeft;
      OwnedConstRoxieRow defaultRight;
      Owned<IEngineRowAllocator> defaultLeftAllocator;
      Owned<IEngineRowAllocator> defaultRightAllocator;
      Owned<IGroupedInput> sortedLeft;
      Owned<IGroupedInput> groupedSortedRight;
  
      bool cloneLeft;
  
      void createDefaultLeft()
      {
          if (!defaultLeft)
          {
              if (!defaultLeftAllocator)
                  defaultLeftAllocator.setown(createRowAllocator(input->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultLeftAllocator);
              size32_t thisSize = helper.createDefaultLeft(rowBuilder);
              defaultLeft.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
      void createDefaultRight()
      {
          if (!defaultRight)
          {
              if (!defaultRightAllocator)
                  defaultRightAllocator.setown(createRowAllocator(input1->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
              size32_t thisSize = helper.createDefaultRight(rowBuilder);
              defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
  public:
      CRoxieServerJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _forceSpill)
          : CRoxieServerTwoInputActivity(_ctx, _factory, _probeManager), helper((IHThorJoinArg &)basehelper), forceSpill(_forceSpill)
      {
          // MORE - some of this should be done in factory
          unsigned joinFlags = helper.getJoinFlags();
          leftOuterJoin = (joinFlags & JFleftouter) != 0;
          rightOuterJoin = (joinFlags  & JFrightouter) != 0;
          exclude = (joinFlags  & JFexclude) != 0;
          cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
          getLimitType(joinFlags, limitFail, limitOnFail);
          if (joinFlags & JFslidingmatch) 
          {
              betweenjoin = true;
              collate = helper.queryCompareLeftRightLower();
              collateupper = helper.queryCompareLeftRightUpper();
          }
          else
          {
              betweenjoin = false;
              collate = collateupper = helper.queryCompareLeftRight();
          }
          rightIndex = 0;
          joinCounter = 0;
          state = JSfill;
          matchedLeft = false;
          joinLimit = 0;
          keepLimit = 0;   // wait until ctx available 
          atmostLimit = 0; // wait until ctx available 
          abortLimit = 0;  // wait until ctx available 
          atmostsTriggered = 0;
          assertex((joinFlags & (JFfirst | JFfirstleft | JFfirstright)) == 0);
          left = NULL;
          pendingRight = NULL;
          activityKind = _factory->getKind();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          left = NULL;
          rightIndex = 0;
          joinCounter = 0;
          state = JSfill;
          matchedLeft = false;
  
          CRoxieServerTwoInputActivity::start(parentExtractSize, parentExtract, paused);
  
          keepLimit = helper.getKeepLimit();
          if (keepLimit == 0) 
              keepLimit = (unsigned)-1;
          atmostsTriggered = 0;
          atmostLimit = helper.getJoinLimit();
          if(atmostLimit == 0)
              atmostLimit = (unsigned)-1;
          else
              assertex(!rightOuterJoin && !betweenjoin);
          abortLimit = helper.getMatchAbortLimit();
          if (abortLimit == 0) 
              abortLimit = (unsigned)-1;
          if (rightOuterJoin)
              createDefaultLeft();
          if ((leftOuterJoin && (activityKind==TAKjoin || activityKind==TAKjoinlight || activityKind==TAKdenormalizegroup)) || limitOnFail)
              createDefaultRight();
          bool isStable = (helper.getJoinFlags() & JFunstable) == 0;
          RoxieSortAlgorithm sortAlgorithm;
          if (forceSpill)
              sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm;
          else
              sortAlgorithm = isStable ? stableQuickSortAlgorithm : quickSortAlgorithm;
          if (helper.isLeftAlreadySorted())
              sortedLeft.setown(createDegroupedInputReader(inputStream));
          else
              sortedLeft.setown(createSortedInputReader(inputStream, createSortAlgorithm(sortAlgorithm, helper.queryCompareLeft(), ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId)));
          ICompare *compareRight = helper.queryCompareRight();
          if (helper.isRightAlreadySorted())
              groupedSortedRight.setown(createGroupedInputReader(inputStream1, compareRight));
          else
              groupedSortedRight.setown(createSortedGroupedInputReader(inputStream1, compareRight, createSortAlgorithm(sortAlgorithm, compareRight, ctx->queryRowManager(), input1->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId)));
          if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit())
          {   //limited match join (s[1..n])
              limitedhelper.setown(createRHLimitedCompareHelper());
              limitedhelper->init( helper.getJoinLimit(), groupedSortedRight, collate, helper.queryPrefixCompare() );
          }
      }
  
      virtual void reset()
      {
          if (atmostsTriggered)
              noteStatistic(StNumAtmostTriggered, atmostsTriggered);
          right.clear();
          ReleaseClearRoxieRow(left);
          ReleaseClearRoxieRow(pendingRight);
          defaultRight.clear();
          defaultLeft.clear();
          sortedLeft.clear();
          groupedSortedRight.clear();
          CRoxieServerTwoInputActivity::reset();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (!idx && (helper.getJoinFlags() & JFparallel) != 0)
          {
              puller.setown(new CRoxieServerReadAheadInput(ctx, ctx->queryOptions().parallelJoinPreload));
              puller->setInput(0, _sourceIdx, _in);
              _in = puller;
              _sourceIdx = 0;
          }
          CRoxieServerTwoInputActivity::setInput(idx, _sourceIdx, _in);
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL : this;
      }
  
      void fillLeft()
      {
          matchedLeft = false;
          left = sortedLeft->nextRow(); // NOTE: already degrouped by the IGroupedInput we attached
          if (betweenjoin && left && pendingRight && (collate->docompare(left, pendingRight) >= 0))
              fillRight();
          if (limitedhelper && 0==rightIndex)
          {
              rightIndex = 0;
              joinCounter = 0;
              right.clear();
              matchedRight.kill();
              if (left)
              {
                  limitedhelper->getGroup(right,left);
                  ForEachItemIn(idx, right)
                      matchedRight.append(false);
              }
          }
      }
  
      void fillRight()
      {
          if (limitedhelper)
              return;
          failingLimit.clear();
          if(betweenjoin && left)
          {
              aindex_t start = 0;
              while(right.isItem(start) && (collateupper->docompare(left, right.item(start)) > 0))
                  start++;
              if(start>0)
                  right.clearPart(0, start);
          }
          else
              right.clear();
          rightIndex = 0;
          joinCounter = 0;
          unsigned groupCount = 0;
          const void * next;
          while(true)
          {
              if(pendingRight)
              {
                  next = pendingRight;
                  pendingRight = NULL;
              }
              else
              {
                  next = groupedSortedRight->nextRow();
              }
              if(!rightOuterJoin && next && (!left || (collateupper->docompare(left, next) > 0))) // if right is less than left, and not right outer, can skip group
              {
                  while(next)
                  {
                      ReleaseClearRoxieRow(next);
                      next = groupedSortedRight->nextRow();
                  }
                  continue;
              }
              while(next)
              {
                  if(groupCount==abortLimit)
                  {
                      if(limitFail)
                          failLimit();
                      if (ctx->queryDebugContext())
                          ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                      if(limitOnFail)
                      {
                          assertex(!failingLimit);
                          try
                          {
                              failLimit();
                          }
                          catch(IException * except)
                          {
                              failingLimit.setown(except);
                          }
                          assertex(failingLimit != NULL);
                      }
                      right.append(next);
                      do
                      {
                          next = groupedSortedRight->nextRow();
                          ReleaseRoxieRow(next);
                      } while(next);
                      break;
                  }
                  else if(groupCount==atmostLimit)
                  {
                      atmostsTriggered++;
                      right.clear();
                      groupCount = 0;
                      while(next) 
                      {
                          ReleaseRoxieRow(next);
                          next = groupedSortedRight->nextRow();
                      }
                  }
                  else
                  {
                      right.append(next);
                      groupCount++;
                  }
                  next = groupedSortedRight->nextRow();
              }
              // normally only want to read one right group, but if is between join and next right group is in window for left, need to continue
              if(betweenjoin && left)
              {
                  pendingRight = groupedSortedRight->nextRow();
                  if(!pendingRight || (collate->docompare(left, pendingRight) < 0))
                      break;
              }
              else
                  break;
          }
          matchedRight.kill();
          ForEachItemIn(idx, right)
              matchedRight.append(false);
      }
  
      const void * joinRecords(const void * curLeft, const void * curRight, unsigned counter, unsigned flags)
      {
          if (cloneLeft)
          {
              LinkRoxieRow(curLeft);
              return curLeft;
          }
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              size32_t thisSize = helper.transform(rowBuilder, curLeft, curRight, counter, flags);
              if (thisSize)
                  return rowBuilder.finalizeRowClear(thisSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * denormalizeRecords(const void * curLeft, ConstPointerArray & rows, unsigned flags)
      {
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned numRows = rows.ordinality();
              const void * right = numRows ? rows.item(0) : defaultRight.get();
              if (numRows>0)
                  flags |= JTFmatchedright;
              size32_t thisSize = helper.transform(rowBuilder, curLeft, right, numRows, (const void * *)rows.getArray(), flags);
              if (thisSize)
                  return rowBuilder.finalizeRowClear(thisSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * joinException(const void * curLeft, IException * except)
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t thisSize = helper.onFailTransform(rowBuilder, curLeft, defaultRight, except, JTFmatchedleft);
          return rowBuilder.finalizeRowClear(thisSize);
      }
  
      void failLimit()
      {
          helper.onMatchAbortLimitExceeded();
          CommonXmlWriter xmlwrite(0);
          if (input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
          {
              input->queryOutputMeta()->toXML((byte *) left, xmlwrite);
          }
          throw MakeStringException(ROXIE_TOO_MANY_RESULTS, "More than %d match candidates in join %d for row %s", abortLimit, queryId(), xmlwrite.str());
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              switch (state)
              {
              case JSfill:
                  fillLeft();
                  state = JSfillright;
                  break;
  
              case JSfillright:
                  fillRight();
                  state = JScollate;
                  break;
  
              case JSfillleft:
                  fillLeft();
                  state = JScollate;
                  break;
  
              case JScollate:
                  if (right.ordinality() == 0)
                  {
                      if (left == NULL)
                          return NULL;
                      state = JSleftonly;
                  }
                  else
                  {
                      if (!left)
                          state = JSrightonly;
                      else
                      {
                          int diff;
                          if(betweenjoin)
                              diff = ((collate->docompare(left, right.item(0)) < 0) ? -1 : ((collateupper->docompare(left, right.item(right.ordinality()-1)) > 0) ? +1 : 0));
                          else
                              diff = collate->docompare(left, right.item(0));
                          bool limitExceeded =  right.ordinality()>abortLimit;
                          if (diff == 0)
                          {
                              if (limitExceeded)
                              {
                                  const void * ret = NULL;
                                  if(failingLimit)
                                      ret = joinException(left, failingLimit);
                                  ReleaseRoxieRow(left);
                                  left = NULL;
                                  state = JSfillleft;
                                  ForEachItemIn(idx, right)
                                      matchedRight.replace(true, idx);
                                  if(ret)
                                  {
                                      processed++;
                                      return ret;
                                  }
                              }
                              else
                              {
                                  state = JScompare;
                                  joinLimit = keepLimit;
                              }
                          }
                          else if (diff < 0)
                              state = JSleftonly;
                          else if (limitExceeded)
                          {
                              // MORE - Roxie code seems to think there should be a destroyRowset(right) here....
                              state = JSfillright;
                          }
                          else
                              state = JSrightonly;
                      }
                  }
                  break;
  
              case JSrightonly:
                  if (rightOuterJoin)
                  {
                      switch (activityKind)
                      {
                      case TAKjoin:
                          {
                              while (right.isItem(rightIndex))
                              {
                                  if (!matchedRight.item(rightIndex))
                                  {
                                      const void * rhs = right.item(rightIndex++);
                                      const void *ret = joinRecords(defaultLeft, rhs, 0, JTFmatchedright);
                                      if (ret)
                                      {
                                          processed++;
                                          return ret;
                                      }
                                  }
                                  else
                                      rightIndex++;
                              }
                              break;
                          }
                      //Probably excessive to implement the following, but possibly useful
                      case TAKdenormalize:
                          {
                              OwnedConstRoxieRow newLeft;
                              newLeft.set(defaultLeft);
                              unsigned rowSize = 0;
                              unsigned leftCount = 0;
                              while (right.isItem(rightIndex))
                              {
                                  if (!matchedRight.item(rightIndex))
                                  {
                                      const void * rhs = right.item(rightIndex);
                                      try
                                      {
                                          RtlDynamicRowBuilder rowBuilder(rowAllocator);
                                          unsigned thisSize = helper.transform(rowBuilder, newLeft, rhs, ++leftCount, JTFmatchedright);
                                          if (thisSize)
                                          {
                                              rowSize = thisSize;
                                              newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
                                          }
                                      }
                                      catch (IException *E)
                                      {
                                          throw makeWrappedException(E);
                                      }
                                  }
                                  rightIndex++;
                              }
                              state = JSfillright;
                              if (rowSize)
                              {
                                  processed++;
                                  return newLeft.getClear();
                              }
                              break;
                          }
                      case TAKdenormalizegroup:
                          {
                              filteredRight.kill();
                              while (right.isItem(rightIndex))
                              {
                                  if (!matchedRight.item(rightIndex))
                                      filteredRight.append(right.item(rightIndex));
                                  rightIndex++;
                              }
                              state = JSfillright;
                              if (filteredRight.ordinality())
                              {
                                  const void * ret = denormalizeRecords(defaultLeft, filteredRight, 0);
                                  filteredRight.kill();
  
                                  if (ret)
                                  {
                                      processed++;
                                      return ret;
                                  }
                              }
                              break;
                          }
                      }
                  }
                  state = JSfillright;
                  break;
                  
              case JSleftonly:
              {
                  const void * ret = NULL;
                  if (!matchedLeft && leftOuterJoin)
                  {
                      switch (activityKind)
                      {
                      case TAKjoin:
                          ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
                          break;
                      case TAKdenormalize:
                          ret = left;
                          left = NULL;
                          break;
                      case TAKdenormalizegroup:
                          filteredRight.kill();
                          ret = denormalizeRecords(left, filteredRight, JTFmatchedleft);
                          break;
                      }
                  }
                  ReleaseRoxieRow(left);
                  left = NULL;
                  state = JSfillleft;
                  if (ret)
                  {
                      processed++;
                      return ret;
                  }
                  break;
              }
  
              case JScompare:
                  if (joinLimit != 0)
                  {
                      switch (activityKind)
                      {
                      case TAKjoin:
                          {
                              while (right.isItem(rightIndex))
                              {
                                  const void * rhs = right.item(rightIndex++);
                                  if (helper.match(left, rhs))
                                  {
                                      matchedRight.replace(true, rightIndex-1);
                                      matchedLeft = true;
                                      if (!exclude)
                                      {
                                          const void *ret = joinRecords(left, rhs, ++joinCounter, JTFmatchedleft|JTFmatchedright);
                                          if (ret)
                                          {
                                              processed++;
                                              joinLimit--;
                                              return ret;
                                          }
                                      }
                                  }
                              }
                              break;
                          }
                      case TAKdenormalize:
                          {
                              OwnedConstRoxieRow newLeft;
                              newLeft.set(left);
                              unsigned rowSize = 0;
                              unsigned leftCount = 0;
                              while (right.isItem(rightIndex) && joinLimit)
                              {
                                  try
                                  {
                                      const void * rhs = right.item(rightIndex++);
                                      if (helper.match(left, rhs))
                                      {
                                          matchedRight.replace(true, rightIndex-1);
                                          matchedLeft = true;
                                          if (!exclude)
                                          {
                                              RtlDynamicRowBuilder rowBuilder(rowAllocator);
                                              unsigned thisSize = helper.transform(rowBuilder, newLeft, rhs, ++leftCount, JTFmatchedleft|JTFmatchedright);
                                              if (thisSize)
                                              {
                                                  rowSize = thisSize;
                                                  newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
                                                  joinLimit--;
                                              }
                                          }
                                      }
                                  }
                                  catch (IException *E)
                                  {
                                      throw makeWrappedException(E);
                                  }
                              }
                              state = JSleftonly;
                              rightIndex = 0;
                              if (rowSize)
                              {
                                  processed++;
                                  return newLeft.getClear();
                              }
                              break;
                          }
                      case TAKdenormalizegroup:
                          {
                              filteredRight.kill();
                              while (right.isItem(rightIndex))
                              {
                                  const void * rhs = right.item(rightIndex++);
                                  if (helper.match(left, rhs))
                                  {
                                      matchedRight.replace(true, rightIndex-1);
                                      filteredRight.append(rhs);
                                      matchedLeft = true;
                                      if (filteredRight.ordinality()==joinLimit)
                                          break;
                                  }
                              }
                              state = JSleftonly;
                              rightIndex = 0;
  
                              if (!exclude && filteredRight.ordinality())
                              {
                                  const void * ret = denormalizeRecords(left, filteredRight, JTFmatchedleft);
                                  filteredRight.kill();
  
                                  if (ret)
                                  {
                                      processed++;
                                      return ret;
                                  }
                              }
                              break;
                          }
                      }
                  }
                  state = JSleftonly;
                  rightIndex = 0;
                  joinCounter = 0;
                  break;
              }
          }
      }
  };
  
  class CRoxieServerJoinActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned input2;
      unsigned input2idx;
      bool forceSpill;
  
  public:
      CRoxieServerJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, joinStatistics)
      {
          forceSpill = _graphNode.getPropBool("hint[@name='spill']/@value", _queryFactory.queryOptions().allSortsMaySpill);
          input2 = 0;
          input2idx = 0;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerJoinActivity(_ctx, this, _probeManager, forceSpill);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          if (idx==1)
          {
              input2 = source;
              input2idx = sourceidx;
          }
          else
              CRoxieServerActivityFactory::setInput(idx, source, sourceidx);
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          switch (idx)
          {
          case 1:
              sourceidx = input2idx;
              return input2;
          case 0:
              return CRoxieServerActivityFactory::getInput(idx, sourceidx);
          default:
              return (unsigned) -1; 
          }
      }
  
      virtual unsigned numInputs() const { return 2; }
  };
  
  IRoxieServerActivityFactory *createRoxieServerJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  #define CONCAT_READAHEAD 1000
  
  class CRoxieThreadedConcatReader : implements IRecordPullerCallback, public CInterface
  {
  public:
      IMPLEMENT_IINTERFACE;
      CRoxieThreadedConcatReader(InterruptableSemaphore &_ready, bool _grouped)
      : puller(false), ready(_ready), atEog(true), eof(false), grouped(_grouped)
      {
      }
  
      void start(unsigned parentExtractSize, const byte *parentExtract, bool paused, IRoxieAgentContext *ctx)
      {
          space.reinit(CONCAT_READAHEAD);
          puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().concatPreload, false, ctx);
      }
  
      void stop()
      {
          space.interrupt();
          puller.stop();
      }
  
      IFinalRoxieInput *queryInput() const
      {
          return puller.queryInput();
      }
  
      void reset()
      {
          puller.reset();
          ForEachItemIn(idx, buffer)
              ReleaseRoxieRow(buffer.item(idx));
          buffer.clear();
          eof = false;
          atEog = true;
      }
  
      void setInput(unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          puller.setInput(this, _sourceIdx, _in);
      }
  
      void connectInputStreams(IRoxieAgentContext *ctx, bool consumerOrdered)
      {
          puller.connectInputStreams(ctx, consumerOrdered);
      }
  
      virtual void processRow(const void *row)
      {
          buffer.enqueue(row);
          ready.signal();
          space.wait();
      }
  
      virtual void processGroup(const ConstPointerArray &rows)
      {
          // We use record-by-record input mode of the puller thread even in grouped mode.
          throwUnexpected();
      }
  
      virtual void processEOG()
      {
          if (grouped)
              processRow(NULL);
      }
  
      virtual void processDone()
      {
          processRow(NULL);
      }
  
      virtual bool fireException(IException *e)
      {
          // called from puller thread on failure
          ready.interrupt(LINK(e));
          space.interrupt(e);
          return true;
      }
  
      bool peek(const void * &row, bool &anyActive)
      {
          if (!eof)
          {
              if (buffer.ordinality())
              {
                  space.signal();
                  row = buffer.dequeue();
                  if (row==NULL)
                  {
                      if (atEog)
                      {
                          eof = true;
                          return false;
                      }
                      else
                          atEog = true;
                  }
                  else if (grouped)
                      atEog = false;
                  return true;
              }
              anyActive = true;
          }
          return false;
      }
  
  protected:
      RecordPullerThread puller;
      InterruptableSemaphore space;
      InterruptableSemaphore &ready;
      SafeQueueOf<const void, true> buffer;
      bool atEog;
      bool eof;
      bool grouped;
  };
  
  typedef CopyReferenceArrayOf<CRoxieThreadedConcatReader> ReaderArray;
  
  class CRoxieServerThreadedConcatActivity : public CRoxieServerActivity
  {
      InterruptableSemaphore ready;
      ReaderArray pullers;
      unsigned numInputs;
      unsigned nextPuller; // for round robin
      unsigned readyPending;
      bool eof;
      bool inGroup;
      bool grouped;
  
  public:
      CRoxieServerThreadedConcatActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _grouped, unsigned _numInputs)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), grouped(_grouped)
      {
          numInputs = _numInputs;
          eof = (numInputs==0);
          inGroup = false;
          nextPuller = 0;
          readyPending = 0;
          for (unsigned i = 0; i < numInputs; i++)
              pullers.append(*new CRoxieThreadedConcatReader(ready, _grouped));
  
      }
  
      ~CRoxieServerThreadedConcatActivity()
      {
          ForEachItemIn(idx, pullers)
              delete &pullers.item(idx);
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = (numInputs==0);
          inGroup = false;
          nextPuller = 0;
          readyPending = 0;
          ready.reinit();
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          ForEachItemIn(idx, pullers)
          {
              pullers.item(idx).start(parentExtractSize, parentExtract, paused, ctx);
              // NOTE - it is ok to start the thread running while parts of the subgraph are still being started, since everything 
              // in the part of the subgraph that the thread uses has been started.
              // Note that splitters are supposed to cope with being used when only some outputs have been started.
          }
      }
  
      virtual void stop()
      {
          ready.interrupt();
          ForEachItemIn(idx, pullers)
              pullers.item(idx).stop();
          CRoxieServerActivity::stop();
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          return 0;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (pullers.isItem(idx))
              return pullers.item(idx).queryInput();
          else
              return NULL;
      }
  
      virtual void reset()
      {
          CRoxieServerActivity::reset();
          ForEachItemIn(idx, pullers)
              pullers.item(idx).reset();
          eof = false;
          inGroup = false;
          nextPuller = 0;
          readyPending = 0;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (pullers.isItem(idx))
              pullers.item(idx).setInput(_sourceIdx, _in);
          else
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          ForEachItemIn(i, pullers)
              pullers.item(i).connectInputStreams(ctx, consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, nullptr, consumerOrdered, nullptr); // No input actually connected
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          for (;;)
          {
              if (readyPending && !inGroup)
              {
                  if (readyPending > 1)
                      ready.signal(readyPending-1);
                  readyPending = 0;
              }
              else
                  ready.wait();
              bool anyActive = false;
              ForEachItemIn(unused_index, pullers)
              {
                  // NOTE - we round robin not just because it's more efficient, but because it ensures the preservation of grouping information
                  const void *ret;
                  bool fetched = pullers.item(nextPuller).peek(ret, anyActive);
                  if (fetched)
                  {
                      inGroup = (ret != NULL);
                      return ret;
                  }
                  if (inGroup && grouped)
                  {
                      // Some other puller has data, but we can't consume it until the group we are reading is complete.
                      readyPending++;
                      anyActive = true;
                      break;
                  }
                  nextPuller++;
                  if (nextPuller==pullers.ordinality())
                      nextPuller = 0;
              }
              if (!anyActive)
                  break;
              // A ready signal without anything being ready means someone reached end-of-file.
          }
          eof = true;
          return NULL;
      }
  
  };
  
  class CRoxieServerOrderedConcatActivity : public CRoxieServerMultiInputActivity
  {
      IEngineRowStream *curStream;
      bool eogSeen;
      bool anyThisGroup;
      bool grouped;
      unsigned streamIdx;
  
  public:
      CRoxieServerOrderedConcatActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _grouped, unsigned _numInputs)
          : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs)
      {
          eogSeen = false;
          anyThisGroup = false;
          grouped = _grouped;
          streamIdx = 0;
          curStream = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
          streamIdx = 0;
          curStream = streamArray[streamIdx];
          eogSeen = false;
          anyThisGroup = false;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!curStream)
              return NULL;  // eof
          const void * next = curStream->nextRow();
          if (next)
          {
              anyThisGroup = true;
              eogSeen = false;
              processed++;
              return next;
          }
          else if (!eogSeen)
          {
              eogSeen = true;
              if (grouped)
              {
                  if (anyThisGroup)
                  {
                      anyThisGroup = false;
                      return NULL;
                  }
                  else
                      return nextRow();
              }
              else
                  return nextRow();
          }
          else if (streamIdx < numStreams-1)
          {
              streamIdx++;
              curStream = streamArray[streamIdx];
              eogSeen = false;
              return nextRow();
          }
          else
          {
              curStream = NULL;
              return NULL;
          }
      }
  };
  
  class CRoxieServerConcatActivityFactory : public CRoxieServerMultiInputFactory
  {
      bool ordered;
      bool grouped;
  
  public:
      CRoxieServerConcatActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned <IHThorFunnelArg> helper = (IHThorFunnelArg *) helperFactory();
          ordered = helper->isOrdered();
          grouped = helper->queryOutputMeta()->isGrouped();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (ordered || (_probeManager && _probeManager->queryDebugManager()))
              return new CRoxieServerOrderedConcatActivity(_ctx, this, _probeManager, grouped, numInputs());
          else
              return new CRoxieServerThreadedConcatActivity(_ctx, this, _probeManager, grouped, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerConcatActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerConcatActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNonEmptyActivity : public CRoxieServerMultiInputBaseActivity
  {
      IEngineRowStream* selectedStream;
      unsigned savedParentExtractSize;
      const byte * savedParentExtract;
      bool foundInput;
      bool unusedStopped;
  
  public:
      CRoxieServerNonEmptyActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs)
      {
          foundInput = false;
          unusedStopped = false;
          selectedStream = NULL;
          savedParentExtractSize = 0;;
          savedParentExtract = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          //Don't start the inputs yet so we can short-circuit...
          CRoxieServerMultiInputBaseActivity::start(parentExtractSize, parentExtract, paused);
          savedParentExtractSize = parentExtractSize;
          savedParentExtract = parentExtract;
      }
  
      virtual void stop()
      {
          if (unusedStopped)
          {
              if (selectedStream)
                  selectedStream->stop();
          }
          else
          {
              //May possibly call stop too many times if input->start() throws an exception (e.g. due to a dependency)
              for (unsigned i = 0; i < numStreams; i++)
                  streamArray[i]->stop();
          }
          CRoxieServerMultiInputBaseActivity::stop();
      }
  
      virtual void reset()    
      {
          CRoxieServerMultiInputBaseActivity::reset(); 
          foundInput = false;
          unusedStopped = false;
          selectedStream = NULL;
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          return 0; // Can't easily calculate anything reliable but local processing is negligible
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!foundInput)
          {
              foundInput = true;
              assertex(numInputs == numStreams);   // If this ceases to be true we will need to rethink some of this
              //If we get an exception in this loop then stop() will stop any started inputs
              for (unsigned i=0; i < numStreams; i++)
              {
                  selectedStream = streamArray[i];
                  inputArray[i]->start(savedParentExtractSize, savedParentExtract, false);  // Assumes 1:1 mapping streams to inputs
                  startJunction(junctionArray[i]);
                  const void * next = selectedStream->nextRow();
                  if (next)
                  {
                      //Found a row so stop remaining
                      for (unsigned j=i+1; j < numInputs; j++)
                          streamArray[j]->stop();
                      unusedStopped = true;
                      processed++;
                      return next;
                  }
                  selectedStream->stop();
              }
              unusedStopped = true;
              selectedStream = NULL;
              return NULL;
          }
          if (!selectedStream)
              return NULL;
          const void * next = selectedStream->nextRow();
          if (next)
              processed++;
          return next;
      }
  };
  
  class CRoxieServerNonEmptyActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerNonEmptyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNonEmptyActivity(_ctx, this, _probeManager, numInputs());
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerNonEmptyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNonEmptyActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class SingleNodeActivityContext : public IThorActivityContext
  {
  public:
      SingleNodeActivityContext(unsigned _numStrands, unsigned _curStrand) : strands(_numStrands), curStrand(_curStrand) { assertex(curStrand < strands); }
  
      virtual bool isLocal() const override { return false; }
      virtual unsigned numSlaves() const override { return 1; }
      virtual unsigned numStrands() const override { return strands; }
      virtual unsigned querySlave() const override { return 0; }
      virtual unsigned queryStrand() const override { return curStrand; }
  protected:
      unsigned strands;
      unsigned curStrand;
  };
  
  class CRoxieServerExternalActivity : public CRoxieServerMultiInputActivity
  {
      Owned<IRowStream> rows;
      SingleNodeActivityContext activityContext;
  
  public:
      CRoxieServerExternalActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs), activityContext(1, 0)
      {
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks) override
      {
          Owned<IStrandJunction> junction = CRoxieServerMultiInputActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
          associateInputsWithHelper();
          return junction.getClear();
      }
  
      virtual void connectInputStreams(bool consumerOrdered)
      {
          CRoxieServerMultiInputActivity::connectInputStreams(consumerOrdered);
          associateInputsWithHelper();
      }
  
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
          if (factory->getKind() != TAKexternalsink)
          {
              IHThorExternalArg & helper = static_cast<IHThorExternalArg &>(basehelper);
              rows.setown(helper.createOutput(&activityContext));
          }
      }
  
      virtual void stop()
      {
          if (rows)
          {
              rows->stop();
              rows.clear();
          }
          CRoxieServerMultiInputActivity::stop();
      }
  
      virtual void reset()
      {
          rows.clear();
          CRoxieServerMultiInputActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          assertex(rows);
          const void * next = rows->nextRow();
          if (next)
              processed++;
          return next;
      }
  
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract)
      {
          try
          {
              start(parentExtractSize, parentExtract, false);
              assertex(!rows);
              IHThorExternalArg & helper = static_cast<IHThorExternalArg &>(basehelper);
              helper.execute(&activityContext);
              stop();
          }
          catch (IException * E)
          {
              ctx->notifyAbort(E);
              abort();
              throw;
          }
      }
  
  protected:
      void associateInputsWithHelper()
      {
          IHThorExternalArg & helper = static_cast<IHThorExternalArg &>(basehelper);
          for (unsigned i = 0; i < numInputs; i++)
              helper.setInput(i, streamArray[i]);
      }
  };
  
  class CRoxieServerExternalActivityFactory : public CRoxieServerMultiInputFactory
  {
      bool isRoot;
  public:
      CRoxieServerExternalActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerExternalActivity(_ctx, this, _probeManager, numInputs());
      }
  
      virtual bool isSink() const
      {
          return (kind == TAKexternalsink) && isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerExternalActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerExternalActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerMergeActivity : public CRoxieServerMultiInputActivity
  {
      IHThorMergeArg &helper;
      unsigned *mergeheap;
      unsigned activeInputs;
      const void **pending;
      bool first;
      ICompare *compare;
      bool dedup;
  
      void permute()
      {
          assertex(activeInputs == 0);
          for(unsigned i = 0; i < numStreams; i++)
              if(pullInput(i))
                  mergeheap[activeInputs++] = i;
          // the tree structure: element p has children p*2+1 and p*2+2, or element c has parent (unsigned)(c-1)/2
          // the heap property: no element should be smaller than its parent
          // the dedup variant: if(dedup), the top of the heap should also not be equal to either child
          // the method: establish this by starting with the parent of the bottom element and working up to the top element, sifting each down to its correct place
          if (activeInputs >= 2)
              for(unsigned p = (activeInputs-2)/2; p > 0; --p)
                  siftDown(p);
          if(dedup)
              siftDownDedupTop();
          else
              siftDown(0);
      }
  
      void readNext()
      {
          if(!pullInput(mergeheap[0]))
              if(!promote(0))
                  return;
          // we have changed the element at the top of the heap, so need to sift it down to maintain the heap property
          if(dedup)
              siftDownDedupTop();
          else
              siftDown(0);
      }
  
      bool pullInput(unsigned i)
      {
          const void *next = streamArray[i]->ungroupedNextRow();
          pending[i] = next;
          return (next != NULL);
      }
  
      bool promote(unsigned p)
      {
          activeInputs--;
          if(activeInputs == p)
              return false;
          mergeheap[p] = mergeheap[activeInputs];
          return true;
      }
  
      bool siftDown(unsigned p)
      {
          // assuming that all descendants of p form a heap, sift p down to its correct position, and so include it in the heap
          bool nochange = true;
          while(1)
          {
              unsigned c = p*2 + 1;
              if(c >= activeInputs) 
                  return nochange;
              if(c+1 < activeInputs)
              {
                  int childcmp = BuffCompare(c+1, c);
                  if((childcmp < 0) || ((childcmp == 0) && (mergeheap[c+1] < mergeheap[c])))
                      ++c;
              }
              int cmp = BuffCompare(c, p);
              if((cmp > 0) || ((cmp == 0) && (mergeheap[c] > mergeheap[p])))
                  return nochange;
              nochange = false;
              unsigned r = mergeheap[c];
              mergeheap[c] = mergeheap[p];
              mergeheap[p] = r;
              p = c;
          }
      }
  
      void siftDownDedupTop()
      {
          // same as siftDown(0), except that it also ensures that the top of the heap is not equal to either of its children
          if(activeInputs < 2)
              return;
          unsigned c = 1;
          int childcmp = 1;
          if(activeInputs >= 3)
          {
              childcmp = BuffCompare(2, 1);
              if(childcmp < 0)
                  c = 2;
          }
          int cmp = BuffCompare(c, 0);
          if(cmp > 0)
              return;
          // the following loop ensures the correct property holds on the smaller branch, and that childcmp==0 iff the top matches the other branch
          while(cmp <= 0)
          {
              if(cmp == 0)
              {
                  if(mergeheap[c] < mergeheap[0])
                  {
                      unsigned r = mergeheap[c];
                      mergeheap[c] = mergeheap[0];
                      mergeheap[0] = r;
                  }
                  ReleaseClearRoxieRow(pending[mergeheap[c]]);
                  if(!pullInput(mergeheap[c]))
                      if(!promote(c))
                          break;
                  siftDown(c);
              }
              else
              {
                  unsigned r = mergeheap[c];
                  mergeheap[c] = mergeheap[0];
                  mergeheap[0] = r;
                  if(siftDown(c))
                      break;
              }
              cmp = BuffCompare(c, 0);
          }
          // the following loop ensures the uniqueness property holds on the other branch too
          c = 3-c;
          if(activeInputs <= c)
              return;
          while(childcmp == 0)
          {
              if(mergeheap[c] < mergeheap[0])
              {
                  unsigned r = mergeheap[c];
                  mergeheap[c] = mergeheap[0];
                  mergeheap[0] = r;
              }
              ReleaseClearRoxieRow(pending[mergeheap[c]]);
              if(!pullInput(mergeheap[c]))
                  if(!promote(c))
                      break;
              siftDown(c);
              childcmp = BuffCompare(c, 0);
          }
      }
  
      inline int BuffCompare(unsigned a, unsigned b)
      {
          return compare->docompare(pending[mergeheap[a]], pending[mergeheap[b]]);
      }
  
  public:
      CRoxieServerMergeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs), helper((IHThorMergeArg &)basehelper)
      {
          activeInputs = 0;
          first = true;
          mergeheap = new unsigned[numInputs];
          pending = new const void *[numStreams];
          compare = helper.queryCompare();
          dedup = helper.dedup();
          for (unsigned i = 0; i < numStreams; i++)
              pending[i] = NULL;
      }
  
      ~CRoxieServerMergeActivity()
      {
          delete [] mergeheap;
          delete [] pending;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          activeInputs = 0;
          first = true;
          CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()    
      {
          for (unsigned i = 0; i < numStreams; i++)
              ReleaseClearRoxieRow(pending[i]);
          CRoxieServerMultiInputActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (first)
          {
              permute();
              first = false;
          }
          if (activeInputs)
          {
              const void *next = pending[mergeheap[0]];
              readNext();
              if (next)
                  processed++;
              return next;
          }
          else
              return NULL;
      }
  };
  
  class CRoxieServerMergeActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerMergeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerMergeActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerMergeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerMergeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerRegroupActivity : public CRoxieServerMultiInputActivity
  {
      unsigned streamIndex;
      bool eof;
      unsigned __int64 numProcessedLastGroup;
  
  public:
      CRoxieServerRegroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs)
      {
          streamIndex = 0;
          eof = false;
          numProcessedLastGroup = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          streamIndex = 0;
          eof = false;
          numProcessedLastGroup = processed;
          CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      const void * nextFromInputs()
      {
          unsigned initialStream = streamIndex;
          while (streamIndex < numStreams)
          {
              const void * next = streamArray[streamIndex]->nextRow();
              if (next)
              {
                  if ((streamIndex != initialStream) && (streamIndex != initialStream+1))
                  {
                      ReleaseRoxieRow(next);
                      throw MakeStringException(ROXIE_MISMATCH_GROUP_ERROR, "Mismatched groups supplied to Regroup (%d)", factory->queryId());
                  }
                  return next;
              }
              streamIndex++;
          }
  
          if ((initialStream != 0) && (initialStream+1 != numStreams))
              throw MakeStringException(ROXIE_MISMATCH_GROUP_ERROR, "Mismatched groups supplied to Regroup (%d)", factory->queryId());
  
          streamIndex = 0;
          return NULL;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          const void * ret = nextFromInputs();
          if (ret)
          {
              processed++;
              return ret;
          }
  
          if (numProcessedLastGroup != processed)
          {
              numProcessedLastGroup = processed;
              return NULL;
          }
  
          eof = true;
          return NULL;
      }
  };
  
  class CRoxieServerRegroupActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerRegroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerRegroupActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerRegroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerRegroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerCombineActivity : public CRoxieServerMultiInputActivity
  {
      IHThorCombineArg &helper;
      unsigned __int64 numProcessedLastGroup;
  
  public:
      CRoxieServerCombineActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs), helper((IHThorCombineArg &)basehelper)
      {
          numProcessedLastGroup = 0;
      }
  
      ~CRoxieServerCombineActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = processed;
          CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      void nextInputs(ConstPointerArray & out)
      {
          for (unsigned i=0; i < numStreams; i++)
          {
              const void * next = streamArray[i]->nextRow();
              if (next)
                  out.append(next);
          }
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              ConstPointerArray group;
              nextInputs(group);
              if ((group.ordinality() == 0) && (numProcessedLastGroup == processed))
                  nextInputs(group);
              if (group.ordinality() == 0)
              {
                  numProcessedLastGroup = processed;
                  return NULL;
              }
              else if (group.ordinality() != numInputs)
              {
                  ReleaseRoxieRows(group);
                  throw MakeStringException(ROXIE_MISMATCH_GROUP_ERROR, "Mismatched group input for Combine Activity(%d)", factory->queryId());
              }
  
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, group.ordinality(), group.getArray());
                  ReleaseRoxieRows(group);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException *E)
              {
                  ReleaseRoxieRows(group);
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerCombineActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerCombineActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerCombineActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerCombineActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerCombineActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerCombineGroupActivity : public CRoxieServerTwoInputActivity
  {
      IHThorCombineGroupArg &helper;
      unsigned __int64 numProcessedLastGroup;
  
  public:
      CRoxieServerCombineGroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerTwoInputActivity(_ctx, _factory, _probeManager), helper((IHThorCombineGroupArg &)basehelper)
      {
          numProcessedLastGroup = 0;
      }
  
      ~CRoxieServerCombineGroupActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = processed;
          CRoxieServerTwoInputActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL : this;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              const void * left = inputStream->nextRow();
              if (!left && (numProcessedLastGroup == processed))
                  left = inputStream->nextRow();
  
              if (!left)
              {
                  if (numProcessedLastGroup == processed)
                  {
                      const void * nextRight = inputStream1->nextRow();
                      if (nextRight)
                      {
                          ReleaseRoxieRow(nextRight);
                          throw MakeStringException(ROXIE_MISSING_GROUP_ERROR, "Missing LEFT record for Combine Group (%d)", factory->queryId());
                      }
                  }
                  else
                      numProcessedLastGroup = processed;
                  return NULL;
              }
  
              ConstPointerArray group;
              for (;;)
              {
                  const void * in = inputStream1->nextRow();
                  if (!in)
                      break;
                  group.append(in);
              }
  
              if (group.ordinality() == 0)
              {
                  ReleaseRoxieRow(left);
                  ReleaseRoxieRows(group);
                  throw MakeStringException(ROXIE_MISSING_GROUP_ERROR, "Missing RIGHT group for Combine Group (%d)", factory->queryId());
              }
  
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, left, group.ordinality(), (const void * *)group.getArray());
                  ReleaseRoxieRow(left);
                  ReleaseRoxieRows(group);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException *E)
              {
                  ReleaseRoxieRow(left);
                  ReleaseRoxieRows(group);
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerCombineGroupActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned input2;
      unsigned input2idx;
  
  public:
      CRoxieServerCombineGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          input2 = 0;
          input2idx = 0;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerCombineGroupActivity(_ctx, this, _probeManager);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          if (idx==1)
          {
              input2 = source;
              input2idx = sourceidx;
          }
          else
              CRoxieServerActivityFactory::setInput(idx, source, sourceidx);
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          switch (idx)
          {
          case 1:
              sourceidx = input2idx;
              return input2;
          case 0:
              return CRoxieServerActivityFactory::getInput(idx, sourceidx);
          default:
              return (unsigned) -1; 
          }
      }
  
      virtual unsigned numInputs() const { return 2; }
  };
  
  IRoxieServerActivityFactory *createRoxieServerCombineGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerCombineGroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerRollupGroupActivity : public CRoxieServerActivity
  {
      IHThorRollupGroupArg &helper;
      bool eof;
  
  public:
      CRoxieServerRollupGroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorRollupGroupArg &)basehelper)
      {
          eof = false;
      }
  
      ~CRoxieServerRollupGroupActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          for (;;)
          {
              ConstPointerArray group;
  
              for (;;)
              {
                  const void * in = inputStream->nextRow();
                  if (!in)
                      break;
                  group.append(in);
              }
  
              if (group.ordinality() == 0)
              {
                  eof = true;
                  return NULL;
              }
  
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, group.ordinality(), (const void * *)group.getArray());
  
                  ReleaseRoxieRows(group);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException * E)
              {
                  ReleaseRoxieRows(group);
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerRollupGroupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerRollupGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerRollupGroupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerRollupGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerRollupGroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerFilterProjectActivity : public CRoxieServerLateStartActivity
  {
      IHThorFilterProjectArg &helper;
      unsigned numProcessedLastGroup;
      unsigned __int64 recordCount;
  
  public:
      CRoxieServerFilterProjectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerLateStartActivity(_ctx, _factory, _probeManager), helper((IHThorFilterProjectArg &)basehelper)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
      }
  
      ~CRoxieServerFilterProjectActivity()
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
          CRoxieServerLateStartActivity::start(parentExtractSize, parentExtract, paused);
          lateStart(parentExtractSize, parentExtract, helper.canMatchAny()); //sets eof
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          for (;;)
          {
              const void * in = inputStream->nextRow();
              if (!in)
              {
                  recordCount = 0;
                  if (numProcessedLastGroup == processed)
                      in = inputStream->nextRow();
                  if (!in)
                  {
                      numProcessedLastGroup = processed;
                      return NULL;
                  }
              }
  
              try 
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, in, ++recordCount);
                  ReleaseRoxieRow(in);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerFilterProjectActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerFilterProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          //MORE: Code generator needs to indicate if the count is used.
          //optStableInput = false;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerFilterProjectActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerFilterProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerFilterProjectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================
  
  class CRoxieServerStrandedProjectActivity : public CRoxieServerStrandedActivity
  {
      class ProjectProcessor : public StrandProcessor
      {
      protected:
          IHThorProjectArg &helper;
  
      public:
          ProjectProcessor(CRoxieServerActivity &_parent, IEngineRowStream *_inputStream, IHThorProjectArg &_helper)
          : StrandProcessor(_parent, _inputStream, true), helper(_helper)
          {
          }
          virtual const void * nextRow()
          {
              ActivityTimer t(activityStats, timeActivities);
              for (;;)
              {
                  OwnedConstRoxieRow in = inputStream->nextRow();
                  if (!in)
                  {
                      if (numProcessedLastGroup == processed)
                          in.setown(inputStream->nextRow());
                      if (!in)
                      {
                          numProcessedLastGroup = processed;
                          return NULL;
                      }
                  }
  
                  try
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      size32_t outSize;
                      outSize = helper.transform(rowBuilder, in);
                      if (outSize)
                      {
                          processed++;
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                  }
                  catch (IException *E)
                  {
                      throw parent.makeWrappedException(E);
                  }
              }
          }
      };
  
  public:
      CRoxieServerStrandedProjectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const StrandOptions &_strandOptions)
          : CRoxieServerStrandedActivity(_ctx, _factory, _probeManager, _strandOptions)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerStrandedActivity::start(parentExtractSize, parentExtract, paused);
          onStartStrands();
      }
  
      virtual StrandProcessor *createStrandProcessor(IEngineRowStream *instream)
      {
          return new ProjectProcessor(*this, instream, (IHThorProjectArg &) basehelper);
      }
      virtual StrandProcessor *createStrandSourceProcessor(bool inputOrdered) { throwUnexpected(); }
  };
  
  class CRoxieServerProjectActivity : public CRoxieServerActivity
  {
      unsigned numProcessedLastGroup;
      unsigned __int64 recordCount;
      bool count;
  
   public:
      CRoxieServerProjectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _count)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            count(_count)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              OwnedConstRoxieRow in = inputStream->nextRow();
              if (!in)
              {
                  recordCount = 0;
                  if (numProcessedLastGroup == processed)
                      in.setown(inputStream->nextRow());
                  if (!in)
                  {
                      numProcessedLastGroup = processed;
                      return NULL;
                  }
              }
  
              try 
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  size32_t outSize;
                  if (count)
                      outSize = ((IHThorCountProjectArg &) basehelper).transform(rowBuilder, in, ++recordCount);
                  else
                      outSize = ((IHThorProjectArg &) basehelper).transform(rowBuilder, in);
                  if (outSize)
                  {
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerProjectActivityFactory : public CRoxieServerActivityFactory
  {
  protected:
      StrandOptions strandOptions;
      bool count;
  public:
      CRoxieServerProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), strandOptions(_graphNode)
      {
          count = (kind==TAKcountproject || kind==TAKprefetchcountproject);
          optStableInput = count;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          // NOTE - if they explicitly requested numStrands = 1 via  hint, we could use either of the below. For now use the stranded version to ensure it is tested.
          if (kind == TAKproject)  // Not supported on prefetch or count projects
              return new CRoxieServerStrandedProjectActivity(_ctx, this, _probeManager, strandOptions);
          else
              return new CRoxieServerProjectActivity(_ctx, this, _probeManager, count);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerProjectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerPrefetchProjectActivity : public CRoxieServerActivity, implements IRecordPullerCallback
  {
      unsigned numProcessedLastGroup;
      bool eof;
      bool allPulled;
      bool isThreaded;
      unsigned preload;
      unsigned __int64 recordCount;
      IHThorPrefetchProjectArg &helper;
      RecordPullerThread puller;
      InterruptableSemaphore ready;
      InterruptableSemaphore space;
  
      class PrefetchInfo : public CInterface
      {
      public:
          inline PrefetchInfo(IHThorPrefetchProjectArg &helper, const void *_in, unsigned __int64 _recordCount)
          {
              if (helper.preTransform(extract, _in, _recordCount))
              {
                  in.setown(_in);
                  recordCount = _recordCount;
              }
              else
              {
                  ReleaseRoxieRow(_in);
                  recordCount = 0;
              }
          }
          OwnedConstRoxieRow in;
          unsigned __int64 recordCount;
          rtlRowBuilder extract;
      };
      QueueOf<PrefetchInfo, true> pulled;
      CriticalSection pulledCrit;
  
  public:
      CRoxieServerPrefetchProjectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorPrefetchProjectArg &) basehelper),
          puller(false)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
          eof = false;
          allPulled = false;
          isThreaded = (helper.getFlags() & PPFparallel) != 0;
          preload = 0;
      }
  
      ~CRoxieServerPrefetchProjectActivity()
      {
          while (pulled.ordinality())
              ::Release(pulled.dequeue());
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (idx)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
          puller.setInput(this, _sourceIdx, _in);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          puller.connectInputStreams(ctx, consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return puller.queryInput();
          else
              return NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = 0;
          recordCount = 0;
          eof = false;
          allPulled = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          preload = helper.getLookahead();
          if (!preload)
              preload = ctx->queryOptions().prefetchProjectPreload;
          space.reinit(preload);
          ready.reinit();
          puller.start(parentExtractSize, parentExtract, paused, preload, !isThreaded, ctx);
      }
  
      virtual void stop()
      {
          space.interrupt();
          ready.interrupt();
          CRoxieServerActivity::stop();
          puller.stop();
      }
  
      virtual void reset()
      {
          CRoxieServerActivity::reset();
          puller.reset();
          allPulled = false;
          while (pulled.ordinality())
              ::Release(pulled.dequeue());
      }
  
      virtual PrefetchInfo *readNextRecord()
      {
          if (!isThreaded)
          {
              if (!allPulled) // This looks like it's thread unsafe but we are inside the if(!isThreaded) so should be ok
                  puller.pullRecords(1);
          }
          else
              ready.wait();
          CriticalBlock b(pulledCrit);
          PrefetchInfo *ret = pulled.ordinality() ? pulled.dequeue() : NULL;
          space.signal();
          return ret;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          for (;;)
          {
              Owned<PrefetchInfo> in = readNextRecord();
              if (!in)
              {
                  recordCount = 0;
                  if (numProcessedLastGroup == processed)
                      in.setown(readNextRecord());
                  if (!in)
                  {
                      numProcessedLastGroup = processed;
                      eof = true;
                      return NULL;
                  }
              }
  
              try 
              {
                  if (in->in)
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      size32_t outSize;
                      IThorChildGraph *child = helper.queryChild();
                      Owned<IEclGraphResults> results;
                      if (child)
                          results.setown(child->evaluate(in->extract.size(), in->extract.getbytes()));
                      outSize = helper.transform(rowBuilder, in->in, results, in->recordCount);
                      if (outSize)
                      {
                          processed++;
                          return rowBuilder.finalizeRowClear(outSize);
                      }
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  
  // interface IExceptionHandler
      virtual bool fireException(IException *e)
      {
          // called from puller thread on failure
          ready.interrupt(LINK(e));
          space.interrupt(e);
          return true;
      }
  // interface IRecordPullerCallback
      virtual void processRow(const void *row) 
      {
          {
              CriticalBlock b(pulledCrit);
              pulled.enqueue(new PrefetchInfo(helper, row, ++recordCount));
          }
          if (isThreaded)
          {
              ready.signal();
              space.wait();
          }
      }
  
      virtual void processEOG()
      {
          {
              CriticalBlock b(pulledCrit);
              pulled.enqueue(NULL);
          }
          if (isThreaded)
          {
              ready.signal();
              space.wait();
          }
      }
      virtual void processGroup(const ConstPointerArray &rows)
      {
          throwUnexpected();
      }
      virtual void processDone() 
      {
          CriticalBlock b(pulledCrit);
          allPulled = true;
          if (isThreaded)
              ready.signal();
      }
  };
  
  class CRoxieServerPrefetchProjectActivityFactory : public CRoxieServerProjectActivityFactory 
  {
  public:
      CRoxieServerPrefetchProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerProjectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerPrefetchProjectActivity(_ctx, this, _probeManager);
      }
  };
  
  extern IRoxieServerActivityFactory *createRoxieServerPrefetchProjectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerPrefetchProjectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CPointerArrayRoxieInput : public CPseudoRoxieInput
  {
  public:
      CPointerArrayRoxieInput()
      {
          rowset = NULL;
          rowcount = 0;
          curRow = 0;
      }
      void init(size32_t _rowcount, const byte **_rowset)
      {
          rowset = _rowset;
          rowcount = _rowcount;
          curRow = 0;
      }
      virtual const void * nextRow()
      {
          if (curRow < rowcount)
          {
              const void * ret = rowset[curRow];
              if (ret)
                  LinkRoxieRow(ret);
              curRow++;
              return ret;
          }
          return NULL;
      }
  protected:
      const byte **rowset;
      size32_t rowcount;
      size32_t curRow;
  };
  
  class CRoxieServerLoopActivity : public CRoxieServerActivity
  {
  protected:
      IHThorLoopArg &helper;
      ThorActivityKind activityKind;
      unsigned maxIterations;
      bool finishedLooping;
      unsigned flags;
      bool eof;
      rtlRowBuilder loopExtractBuilder;
      unsigned loopGraphId;
      Linked<IOutputMetaData> counterMeta;
  
  public:
      CRoxieServerLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _loopGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorLoopArg &)basehelper), loopGraphId(_loopGraphId), counterMeta(_counterMeta)
      {
          eof = false;
          finishedLooping = false;
          activityKind = factory->getKind();
          flags = helper.getFlags();
          maxIterations = 0;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          int iterations = (int) helper.numIterations();
          maxIterations = (iterations >= 0) ? iterations : 0;
          finishedLooping = ((activityKind == TAKloopcount) && (maxIterations == 0));
          if ((flags & IHThorLoopArg::LFnewloopagain) && !helper.loopFirstTime())
              finishedLooping = true;
          loopExtractBuilder.clear();
          helper.createParentExtract(loopExtractBuilder);         // could possibly delay this until execution actually happens
      }
  
      virtual void stop()
      {
          CRoxieServerActivity::stop();
          loopExtractBuilder.clear();
      }
  
      void createCounterResult(IRoxieServerChildGraph * graph, unsigned counter)
      {
          if (flags & IHThorLoopArg::LFcounter)
          {
              void * counterRow = ctx->queryRowManager().allocate(sizeof(thor_loop_counter_t), activityId);
              *((thor_loop_counter_t *)counterRow) = counter;
              RtlLinkedDatasetBuilder builder(rowAllocator);
              builder.appendOwn(counterRow);
              Owned<CGraphResult> counterResult = new CGraphResult(builder.getcount(), builder.linkrows());
              graph->setInputResult(2, counterResult);
          }
      }
  };
  
  //=================================================================================
  
  class CRoxieServerSequentialLoopActivity : public CRoxieServerLoopActivity
  {
      Owned<IActivityGraph> loopQuery;
      Owned<IRoxieServerChildGraph> loopGraph;
      IEngineRowStream * curStream;
      RtlLinkedDatasetBuilder *loopInputBuilder;
      CPointerArrayRoxieInput arrayInput;
      Linked<IEngineRowStream> resultStream;
      unsigned loopCounter;
  
  public:
      CRoxieServerSequentialLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _loopGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerLoopActivity(_ctx, _factory, _probeManager, _loopGraphId, _counterMeta)
      {
          curStream = NULL;
          loopCounter = 0;
          loopInputBuilder = NULL;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerLoopActivity::onCreate(_colocalParent);
          loopQuery.set(ctx->queryChildGraph(loopGraphId));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          curStream = inputStream;
          loopCounter = 1;
          CRoxieServerLoopActivity::start(parentExtractSize, parentExtract, paused);
  
          //MORE: Not sure about this, should IRoxieServerChildGraph be combined with IActivityGraph?
          loopGraph.set(loopQuery->queryLoopGraph());
          loopInputBuilder = new RtlLinkedDatasetBuilder(rowAllocator);
      }
  
      virtual void stop()
      {
          delete loopInputBuilder;
          loopInputBuilder = NULL;
          CRoxieServerLoopActivity::stop();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          unsigned emptyIterations = 0;
          for (;;)
          {
              for (;;)
              {
                  const void * ret = curStream->nextRow();
                  if (!ret)
                  {
                      ret = curStream->nextRow();      // more cope with groups somehow....
                      if (!ret)
                      {
                          if (finishedLooping)
                          {
                              eof = true;
                              return NULL;
                          }
                          break;
                      }
                  }
  
                  if (finishedLooping || 
                      ((flags & IHThorLoopArg::LFfiltered) && !helper.sendToLoop(loopCounter, ret)))
                  {
                      processed++;
                      return ret;
                  }
                  loopInputBuilder->appendOwn(ret);
              }
  
              switch (activityKind)
              {
              case TAKloopdataset:
                  {
                      if (!(flags & IHThorLoopArg::LFnewloopagain))
                      {
                          if (!helper.loopAgain(loopCounter, loopInputBuilder->getcount(), (const void**) loopInputBuilder->queryrows()))
                          {
                              if (loopInputBuilder->getcount() == 0)
                              {
                                  eof = true;
                                  return NULL;
                              }
                              arrayInput.init(loopInputBuilder->getcount(), loopInputBuilder->linkrows());
                              // MORE - should builder be cleared here?
                              curStream = &arrayInput;
                              finishedLooping = true;
                              continue;       // back to the input loop again
                          }
                      }
                      break;
                  }
              case TAKlooprow:
                  if (!loopInputBuilder->getcount())
                  {
                      finishedLooping = true;
                      eof = true;
                      return NULL;
                  }
                  break;
              }
  
              if (loopInputBuilder->getcount())
                  emptyIterations = 0;
              else
              {
                  //note: any outputs which didn't go around the loop again, would return the record, reinitializing emptyIterations
                  emptyIterations++;
                  if (emptyIterations > maxEmptyLoopIterations)
                      throw MakeStringException(ROXIE_TOO_MANY_EMPTY_LOOP, "Executed LOOP with empty input and output %u times", emptyIterations);
                  if (emptyIterations % 32 == 0)
                      CTXLOG("Executing LOOP with empty input and output %u times", emptyIterations);
              }
  
              checkAbort();
              try 
              {
                  Owned<IRoxieGraphResults> results = executeIteration(loopExtractBuilder.size(), loopExtractBuilder.getbytes(), loopCounter);
                  resultStream.setown(results->createIterator(0));
  
                  if (flags & IHThorLoopArg::LFnewloopagain)
                  {
                      Owned<IEngineRowStream> againResult = results->createIterator(helper.loopAgainResult());
                      OwnedConstRoxieRow row  = againResult->nextRow();
                      assertex(row);
                      //Result is a row which contains a single boolean field.
                      if (!((const bool *)row.get())[0])
                          finishedLooping = true;
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
  
              curStream = resultStream.get();
  
              loopCounter++;
              if ((activityKind == TAKloopcount) && (loopCounter > maxIterations))
                  finishedLooping = true;
          }
      }
  
      IRoxieGraphResults * executeIteration(unsigned parentExtractSize, const byte *parentExtract, unsigned counter)
      {
          try
          {
              loopGraph->beforeExecute();
  
              Owned<IGraphResult> inputRowsResult = new CGraphResult(loopInputBuilder->getcount(), loopInputBuilder->linkrows());
              loopInputBuilder->clear();
              loopGraph->setInputResult(1, inputRowsResult);
  
              createCounterResult(loopGraph, counter);
  
              Owned<IRoxieGraphResults> ret = loopGraph->execute(parentExtractSize, parentExtract);
              loopGraph->afterExecute();
              return ret.getClear();
          }
          catch (...)
          {
              CTXLOG("Exception thrown in loop body - cleaning up");
              loopGraph->afterExecute();
              throw;
          }
      }
  };
  
  //=================================================================================
  
  typedef SafeQueueOf<const void, true> SafeRowQueue;
  
  class CRowQueuePseudoInput : public CPseudoRoxieInput
  {
  public:
      CRowQueuePseudoInput(SafeRowQueue & _input, IOutputMetaData * _meta)
      : input(_input), meta(_meta)
      {
          eof = false;
      }
  
      virtual const void * nextRow()
      {
          if (eof)
              return NULL;
          const void * ret = input.dequeue();
          if (!ret)
              eof = true;
          return ret;
      }
  
      virtual IOutputMetaData * queryOutputMeta() const { return meta; }
  
  protected:
      SafeRowQueue & input;
      IOutputMetaData * meta;
      bool eof;
  };
  
  
  class CRoxieServerParallelLoopActivity;
  class LoopFilterPseudoInput : public CIndirectRoxieInput
  {
  public:
      LoopFilterPseudoInput(CRoxieServerParallelLoopActivity * _activity, unsigned _counter) :
          activity(_activity), counter(_counter)
      {
      }
  
      virtual const void * nextRow();
  
  protected:
      CRoxieServerParallelLoopActivity * activity;
      unsigned counter;
  };
  
  class LoopExecutorThread : public RestartableThread
  {
  protected:
      Owned<CSafeRoxieInput> safeInput;
      CRoxieServerParallelLoopActivity * activity;
      bool eof;
      CriticalSection crit;
      unsigned flags;
      SafeRowQueue tempResults[2];
      unsigned savedParentExtractSize;
      const byte * savedParentExtract;
      IArrayOf<IActivityGraph> cachedGraphs;
      IRoxieAgentContext *ctx;
  
  public:
      LoopExecutorThread() 
          : RestartableThread("LoopExecutorThread")
      {
          activity = NULL;
          eof = false;
          flags = 0;
          ctx = NULL;
          savedParentExtract = NULL;
          savedParentExtractSize = 0;
      }
  
      void setInput(CRoxieServerParallelLoopActivity * _activity, unsigned _sourceIdx, IFinalRoxieInput *_input, unsigned _flags)
      {
          activity = _activity;
          flags = _flags;
          // stop is called on our consumer's thread. We need to take care calling stop for our input to make sure it is not in mid-nextRow etc etc.
          safeInput.setown(new CSafeRoxieInput(_sourceIdx, _input));
      }
  
      void connectInputStreams(bool consumerOrdered)
      {
          safeInput->connectInputStreams(ctx, consumerOrdered);
      }
  
      IFinalRoxieInput *queryInput() const
      {
          return safeInput;
      }
  
      void onCreate(IRoxieAgentContext * _ctx);
  
      void start(unsigned parentExtractSize, const byte *parentExtract, bool paused);
      void stop();
      void reset();
  
      virtual int run();
  
  protected:
      void executeLoop();
      void executeLoopInstance(unsigned counter, unsigned numIterations, unsigned inputIdx, IFinalRoxieInput * input, SafeRowQueue * spillOutput);
      IFinalRoxieInput * createLoopIterationGraph(unsigned i, unsigned inputIdx, IFinalRoxieInput * input, unsigned counter);
  };
  
  
  class CRoxieServerParallelLoopActivity : public CRoxieServerLoopActivity
  {
      friend class LoopFilterPseudoInput;
      friend class LoopExecutorThread;
  
      QueueOf<const void, true> ready;
      CriticalSection helperCS;
      CriticalSection cs;
      unsigned defaultNumParallel;
      LoopExecutorThread executor;
      IProbeManager* probeManager;
      CriticalSection canAccess;
      CriticalSection scrit;
      InterruptableSemaphore readySpace;
      InterruptableSemaphore recordsReady;
  
  protected:
      bool includeInLoop(unsigned counter, const void * row)
      {
          CriticalBlock b(helperCS);
          return helper.sendToLoop(counter, row);
      }
  
  public:
      CRoxieServerParallelLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _loopGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerLoopActivity(_ctx, _factory, _probeManager, _loopGraphId, _counterMeta),
            readySpace(parallelLoopFlowLimit)
      {
          probeManager = _probeManager;
          defaultNumParallel = 0;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          executor.onCreate(ctx);
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CriticalBlock b(scrit); // can stop while still starting, if unlucky...
          readySpace.reinit(parallelLoopFlowLimit);
          recordsReady.reinit();    
          CRoxieServerLoopActivity::start(parentExtractSize, parentExtract, paused);
          defaultNumParallel = helper.defaultParallelIterations();
          if (!defaultNumParallel)
              defaultNumParallel = DEFAULT_PARALLEL_LOOP_THREADS;
  
          //MORE: If numIterations <= number of parallel iterations[1], 
          //then we don't need to create a separate thread to do the processing, and the results will also avoid
          //being transferred via a queue
          executor.start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (idx)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
          executor.setInput(this, _sourceIdx, _in, flags);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          executor.connectInputStreams(consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, nullptr, consumerOrdered, nullptr);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return executor.queryInput();
          else
              return NULL;
      }
  
      virtual void stop()
      {
          CriticalBlock b(scrit); // can stop while still starting, if unlucky...
          readySpace.interrupt();
          recordsReady.interrupt();
          executor.join(); // MORE - may not be needed given stop/reset split
          CRoxieServerLoopActivity::stop();
      }
  
      virtual void reset()
      {
          while (ready.ordinality())
              ReleaseRoxieRow(ready.dequeue());
          executor.reset();
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if (eof)
                  return NULL;
  
              recordsReady.wait();
              CriticalBlock procedure(canAccess);
              if (ready.ordinality())
              {
                  const void *result = ready.dequeue();
                  readySpace.signal();
                  if (result)
                      processed++;
                  return result;
              }
              else
                  eof = true;
          }
      }
  
      unsigned getNumParallel(unsigned iter)
      {
          return defaultNumParallel;
      }
  
      inline void enqueueResult(const void * row)
      {
          try
          {
              while(!readySpace.wait(1000))
              {
                  CTXLOG("Blocked waiting for space in loop %p activity id: %d output queue: %d records in queue", this, queryId(), ready.ordinality());
              }
          }
          catch (...)
          {
              ReleaseRoxieRow(row);
              throw;
          }
  
          CriticalBlock b2(canAccess);
          ready.enqueue(row);
          recordsReady.signal();
      }
  
      inline void finishResults()
      {
          recordsReady.signal();
      }
  
      virtual bool fireException(IException *e)
      {
          readySpace.interrupt(LINK(e));
          recordsReady.interrupt(e);
          return true;
      }
  
      IActivityGraph * createChildGraphInstance()
      {
          return factory->createChildGraph(ctx, &helper, loopGraphId, this, probeManager, *this);
      }
  
      IActivityGraph * queryChildGraph()
      {
          return ctx->queryChildGraph(loopGraphId);
      }
  };
  
  //=================================================================================
  
  const void * LoopFilterPseudoInput::nextRow()
  {
      for (;;)
      {
          const void * next = stream->nextRow();
          if (!next || activity->includeInLoop(counter, next))
              return next;
          activity->enqueueResult(next);
      }
  }
  
  
  void LoopExecutorThread::onCreate(IRoxieAgentContext * _ctx)
  {
      //Initialise the cached graph list with the child instance that will always be created.  Other iterations will be created on demand.
      ctx = _ctx;
      cachedGraphs.append(*LINK(activity->queryChildGraph()));
  }
  
  
  void LoopExecutorThread::start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
  {
      savedParentExtractSize = parentExtractSize;
      savedParentExtract = parentExtract;
      eof = false;
      StringBuffer logPrefix("[");
      ctx->getLogPrefix(logPrefix).append("] ");
      RestartableThread::start(logPrefix);
  }
  
  int LoopExecutorThread::run()
  {
      try
      {
          executeLoop();
      }
      catch (IException *e)
      {
          activity->fireException(e);
      }
      catch (...)
      {
          activity->fireException(MakeStringException(ROXIE_INTERNAL_ERROR, "Unexpected exception caught in LoopExecutorThread::run"));
      }
      return 0;
  }
  
  void LoopExecutorThread::stop()
  {
      safeInput->stop();
      RestartableThread::join();
  }
  
  void LoopExecutorThread::reset()
  {
      safeInput->reset();
  }
  
  void LoopExecutorThread::executeLoop()
  {
      unsigned iterations = 0;
      unsigned counter = 0;
      unsigned outputIndex = 0;
  
      //Note, activities don't link inputs, so need to be careful that special inputs remain linked while the activity is executing.
      for (;;)
      {
          if (activity->activityKind == TAKloopcount)
          {
              if (counter == activity->maxIterations)
                  break;
          }
          else
          {
              //This condition isn't quite right because it needs to be whether the filtered
              //input is empty.  May be ok if we include that in the semantics,
              if (tempResults[1-outputIndex].ordinality() == 0)
                  break;
          }
  
          unsigned numParallel = activity->getNumParallel(iterations);
          Linked<IFinalRoxieInput> curInput;
          if (iterations == 0)
              curInput.set(safeInput);
          else
          {
              SafeRowQueue & inputQueue = tempResults[1-outputIndex];
              inputQueue.enqueue(NULL);
              curInput.setown(new CRowQueuePseudoInput(inputQueue, activity->queryOutputMeta()));
          }
  
          SafeRowQueue * curOutput = NULL;
          if (counter+numParallel > activity->maxIterations)
              numParallel = activity->maxIterations - counter;
          else if (counter+numParallel < activity->maxIterations)
              curOutput = &tempResults[outputIndex];
  
          executeLoopInstance(counter, numParallel, 0, curInput, curOutput);  // CurInput is always SOME sort of proxy so idx always 0
          outputIndex = 1-outputIndex;
          counter += numParallel;
          iterations++;
      }
  
      //Check for TAKlooprow, where end of loop couldn't be determined ahead of time
      SafeRowQueue & inputQueue = tempResults[1-outputIndex];
      while (inputQueue.ordinality())
      {
          const void * next = inputQueue.dequeue();
          activity->enqueueResult(next);
      }
      activity->finishResults();
  }
  
  void LoopExecutorThread::executeLoopInstance(unsigned counter, unsigned numIterations, unsigned inputIdx, IFinalRoxieInput * input, SafeRowQueue * spillOutput)
  {
      IArrayOf<IFinalRoxieInput> savedInputs;              // activities don't link their inputs, so this list keeps filters alive.
      Linked<IFinalRoxieInput> curInput = input;
      unsigned i;
      for (i= 0; i != numIterations; i++)
      {
          unsigned thisCounter = counter+i+1;
          IFinalRoxieInput * filtered = curInput;
          if (flags & IHThorLoopArg::LFfiltered)
          {
              LoopFilterPseudoInput *newFiltered = new LoopFilterPseudoInput(activity, thisCounter);
              newFiltered->setInput(inputIdx, curInput);
              filtered = newFiltered;
              savedInputs.append(*filtered);
              inputIdx = 0;
          }
          //graph is kept, so new curInput will be guaranteed to exist
          curInput.setown(createLoopIterationGraph(i, inputIdx, filtered, thisCounter));
          inputIdx = 0;
      }
  
      Owned<IStrandJunction> curJunction;
      IEngineRowStream *curStream = connectSingleStream(ctx, curInput, inputIdx, curJunction, true);
      Owned<IException> failure;
      try
      {
          curInput->start(savedParentExtractSize, savedParentExtract, false);
          if (spillOutput)
          {
              for (;;)
              {
                  const void * next = curStream->nextRow();
                  if (!next)
                      break;
                  spillOutput->enqueue(next);
              }
          }
          else
          {
              for (;;)
              {
                  const void * next = curStream->nextRow();
                  if (!next)
                      break;
                  activity->enqueueResult(next);
              }
          }
      }
      catch (IException *E)
      {
          ctx->notifyAbort(E);
          failure.setown(E);
      }
      for (i= 0; i != numIterations; i++)
      {
          IRoxieServerChildGraph * loopGraph = cachedGraphs.item(i).queryLoopGraph();
          loopGraph->afterExecute();
          //NOTE: inputIdx is always 0 - the code above could be cleaned up.
          loopGraph->querySetInputResult(1, 0, nullptr);
      }
      curStream->stop();
      curInput->reset();
      if (failure)
          throw failure.getClear();
  }
  
  IFinalRoxieInput * LoopExecutorThread::createLoopIterationGraph(unsigned i, unsigned inputIdx, IFinalRoxieInput * input, unsigned counter)
  {
      if (!cachedGraphs.isItem(i))
          cachedGraphs.append(*activity->createChildGraphInstance());
  
      Linked<IRoxieServerChildGraph> loopGraph = cachedGraphs.item(i).queryLoopGraph();
      loopGraph->beforeExecute();
      if (!loopGraph->querySetInputResult(1, inputIdx, input))
          throwUnexpected();      // a loop which doesn't use the value  from the previous iteration.  Should probably handle even if daft.
  
      activity->createCounterResult(loopGraph, counter);
  
      return loopGraph->selectOutput(0);
  }
  
  
  //=================================================================================
  
  class CCounterRowMetaData : implements IOutputMetaData, public CInterface
  {
  public:
      IMPLEMENT_IINTERFACE
  
      virtual size32_t getRecordSize(const void *)            { return sizeof(thor_loop_counter_t); }
      virtual size32_t getMinRecordSize() const               { return sizeof(thor_loop_counter_t); }
      virtual size32_t getFixedSize() const                   { return sizeof(thor_loop_counter_t); }
      virtual void toXML(const byte * self, IXmlWriter & out) { }
      virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
      virtual unsigned getMetaFlags()                         { return 0; }
      virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
      virtual void destruct(byte * self)  {}
      virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual ISourceRowPrefetcher * createDiskPrefetcher() { return NULL; }
      virtual IOutputMetaData * querySerializedDiskMeta() { return this; }
      virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
      virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
      virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
      virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); } // could provide a static implementation if needed
  };
  
  class CRoxieServerLoopActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned loopGraphId;
      unsigned flags;
  
      Linked<IOutputMetaData> counterMeta;
  
  public:
      CRoxieServerLoopActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _loopGraphId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), loopGraphId(_loopGraphId)
      {
          Owned<IHThorLoopArg> helper = (IHThorLoopArg *) helperFactory();
          flags = helper->getFlags();
          counterMeta.setown(new CCounterRowMetaData);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (flags & IHThorLoopArg::LFparallel)
              return new CRoxieServerParallelLoopActivity(_ctx, this, _probeManager, loopGraphId, counterMeta);
          else
              return new CRoxieServerSequentialLoopActivity(_ctx, this, _probeManager, loopGraphId, counterMeta);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLoopActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _loopGraphId)
  {
      return new CRoxieServerLoopActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _loopGraphId);
  }
  
  //=================================================================================
  
  class CRoxieServerGraphLoopActivity : public CRoxieServerActivity
  {
  protected:
      IHThorGraphLoopArg &helper;
      unsigned maxIterations;
      unsigned flags;
      rtlRowBuilder GraphExtractBuilder;
      unsigned loopGraphId;
      Linked<IOutputMetaData> counterMeta;
  
  public:
      CRoxieServerGraphLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _GraphGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorGraphLoopArg &)basehelper), loopGraphId(_GraphGraphId), counterMeta(_counterMeta)
      {
          flags = helper.getFlags();
          maxIterations = 0;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          int iterations = (int) helper.numIterations();
          maxIterations = (iterations >= 0) ? iterations : 0;
          if (maxIterations > maxGraphLoopIterations)
              throw MakeStringException(ROXIE_TOO_MANY_GRAPH_LOOP, "Attempt to execute graph %u times", maxIterations);
          if (maxIterations != 0)
          {
              GraphExtractBuilder.clear();
              helper.createParentExtract(GraphExtractBuilder);
          }
      }
  
      virtual void stop()
      {
          CRoxieServerActivity::stop();
          GraphExtractBuilder.clear();
      }
  
      void createCounterResult(IRoxieServerChildGraph * graph, unsigned counter)
      {
          if (flags & IHThorGraphLoopArg::GLFcounter)
          {
              void * counterRow = ctx->queryRowManager().allocate(sizeof(thor_loop_counter_t), activityId);
              *((thor_loop_counter_t *)counterRow) = counter;
              RtlLinkedDatasetBuilder builder(rowAllocator);
              builder.appendOwn(counterRow);
              Owned<CGraphResult> counterResult = new CGraphResult(builder.getcount(), builder.linkrows());
              graph->setInputResult(0, counterResult);
          }
      }
  };
  
  //=================================================================================
  
  class CRoxieServerSequentialGraphLoopActivity : public CRoxieServerGraphLoopActivity
  {
      Owned<IActivityGraph> GraphQuery;
      Owned<IRoxieServerChildGraph> loopGraph;
      Linked<IEngineRowStream> resultStream;
      bool evaluated;
  
  public:
      CRoxieServerSequentialGraphLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _GraphGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerGraphLoopActivity(_ctx, _factory, _probeManager, _GraphGraphId, _counterMeta)
      {
          evaluated = false;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerGraphLoopActivity::onCreate(_colocalParent);
          GraphQuery.set(ctx->queryChildGraph(loopGraphId));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerGraphLoopActivity::start(parentExtractSize, parentExtract, paused);
  
          //MORE: Not sure about this, should IRoxieServerChildGraph be combined with IActivityGraph?
          loopGraph.set(GraphQuery->queryLoopGraph());
          evaluated = false;
      }
  
      virtual void stop()
      {
          if (loopGraph)
              loopGraph->clearGraphLoopResults();
          CRoxieServerGraphLoopActivity::stop();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!evaluated)
          {
              executeEntireGraph();
              evaluated = true;
          }
  
          const void * ret = resultStream->nextRow();
          if (ret)
              processed++;
          return ret;
      }
  
      void executeIteration(unsigned parentExtractSize, const byte *parentExtract, unsigned counter)
      {
          try
          {
              loopGraph->beforeExecute();
  
              createCounterResult(loopGraph, counter);
  
              loopGraph->executeGraphLoop(parentExtractSize, parentExtract);
              loopGraph->afterExecute();
          }
          catch (...)
          {
              CTXLOG("Exception thrown in graph body - cleaning up");
              loopGraph->afterExecute();
              throw;
          }
      }
  
      void createInitialGraphInput()
      {
          loopGraph->clearGraphLoopResults();
          RtlLinkedDatasetBuilder builder(rowAllocator);
          inputStream->readAll(builder);
          Owned<CGraphResult> result = new CGraphResult(builder.getcount(), builder.linkrows());
          loopGraph->setGraphLoopResult(0, result);
      }
      void executeEntireGraph()
      {
          createInitialGraphInput();
  
          for (unsigned loopCounter=1; loopCounter <= maxIterations; loopCounter++)
          {
              executeIteration(GraphExtractBuilder.size(), GraphExtractBuilder.getbytes(), loopCounter);
          }
  
          resultStream.setown(loopGraph->getGraphLoopResult(maxIterations));
      }
  };
  
  //=================================================================================
  
  struct GraphOutputSplitterArg : public CThorSplitArg
  {
  public:
      virtual unsigned numBranches()
      {
          return 0;
      }
      virtual IOutputMetaData * queryOutputMeta()
      {
          return NULL;// get it from the parent..
      }
  };
  
  extern "C" IHThorArg * createGraphOutputSplitter() { return new GraphOutputSplitterArg; }
  
  class CGraphIterationInfo : public CInterface
  {
  private:
      Owned<IRoxieServerActivityFactory> factory;     // Note - before sourceAct, so destroyed last
      unsigned sourceIdx;
      Linked<IRoxieServerActivity> sourceAct;
      Linked<IFinalRoxieInput> sourceInput;
      unsigned numUses;
      unsigned iteration;
  
  public:
      CGraphIterationInfo(IRoxieServerActivity * _sourceAct, IFinalRoxieInput *_input, unsigned _sourceIdx, unsigned _iteration)
          : sourceIdx(_sourceIdx), sourceAct(_sourceAct), sourceInput(_input), iteration(_iteration)
      {
          numUses = 0;
      }
  
      inline void noteUsed()
      {
          numUses++;
      }
  
      void createSplitter(IRoxieAgentContext *ctx, IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes)
      {
          if (numUses > 1)
          {
              Owned<IPropertyTree> splitterNode = createPTree(ipt_fast);
              factory.setown(createRoxieServerThroughSpillActivityFactory(sourceAct->queryFactory()->queryQueryFactory(), createGraphOutputSplitter, numUses, *splitterNode));
              IRoxieServerActivity *splitter = factory->createActivity(ctx, NULL);
              splitter->onCreate(NULL);
              IFinalRoxieInput *input = sourceAct->queryOutput(sourceIdx);
              if (probeManager)
              {
                  IRoxieProbe * inputProbe = probeManager->createProbe(static_cast<IInputBase*>(input), sourceAct, splitter, sourceIdx, 0, iteration);
                  probes.append(*LINK(inputProbe));
                  input = &dynamic_cast<IFinalRoxieInput &>(inputProbe->queryInput());
                  sourceIdx = 0;
              }
              sourceAct.setown(splitter);
              sourceAct->setInput(0, sourceIdx, input);
              sourceIdx = 0;
              sourceInput.clear();
          }
      }
  
      IFinalRoxieInput *connectOutput(IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes, IRoxieServerActivity *targetAct, unsigned targetIdx)
      {
          // MORE - not really necessary to create splitters in separate pass, is it?
          if (factory) // we created a splitter....
              sourceInput.set(sourceAct->queryOutput(sourceIdx));
          IFinalRoxieInput *ret = sourceInput;
          if (probeManager)
          {
              IRoxieProbe * inputProbe = probeManager->createProbe(sourceInput, sourceAct, targetAct, sourceIdx, targetIdx, iteration);
              probes.append(*LINK(inputProbe));
              ret = &dynamic_cast<IFinalRoxieInput &>(inputProbe->queryInput());
              // MORE - do we need to connect the stream in the probe?
          }
          if (factory) // we created a splitter....
              sourceIdx++;
          return ret;
      }
  };
  
  
  
  class CRoxieServerParallelGraphLoopActivity : public CRoxieServerGraphLoopActivity, implements IRoxieServerLoopResultProcessor
  {
      Owned<IActivityGraph> childGraph;
      IFinalRoxieInput * resultInput;
      IEngineRowStream * resultStream;
      Owned<IStrandJunction> resultJunction;
      CIArrayOf<CGraphIterationInfo> outputs;
      IArrayOf<IRoxieServerChildGraph> iterationGraphs;
      Owned<CExtractMapperInput> inputExtractMapper;
      IProbeManager *probeManager;
      Owned<IStatisticGatherer> childStats;
      unsigned createLoopCounter;
  
      IArrayOf<IRoxieProbe> probes;
  
  public:
      CRoxieServerParallelGraphLoopActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _GraphGraphId, IOutputMetaData * _counterMeta)
          : CRoxieServerGraphLoopActivity(_ctx, _factory, _probeManager, _GraphGraphId, _counterMeta), probeManager(_probeManager)
      {
          inputExtractMapper.setown(new CExtractMapperInput);
          resultInput = NULL;
          resultStream = NULL;
          createLoopCounter = 0;
  
          if (ctx && ctx->collectingDetailedStatistics())
          {
              StatsScopeId rootScope(SSTnone, 0U);
              childStats.setown(createStatisticsGatherer(SCTroxie, "", rootScope));
          }
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerGraphLoopActivity::onCreate(_colocalParent);
          childGraph.set(ctx->queryChildGraph(loopGraphId));
      }
  
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override
      {
          CRoxieServerGraphLoopActivity::gatherStatistics(statsBuilder);
          if (statsBuilder && childStats)
          {
              //Merge any stats collected from executed graphs into the full statistics
              Owned<IStatisticCollection> childCollection = childStats->getResult();
              childCollection->mergeInto(*statsBuilder);
          }
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          //Input needs to be handled very carefully.....
          //We don't want to call onStart on the input unless it is actually used, so don't use the base CRoxieServerActivity implementation.
          //This activity's input needs to be started with (parentExtractSize, parentExtract), but the elements in the graph need to be started with the
          //GraphExtractBuilder parent extract.  So we need to wrap the input in a pseudo-input (inputExtractMapper) that passes through a different 
          //parentExtract.  Something very similar will be needed for query library calls with streaming inputs when they are implemented.
          assertex(idx == 0);
          inputExtractMapper->setInput(_sourceIdx, _in);
      }
  
  
      virtual void connectInputStreams(bool consumerOrdered)
      {
          //NB: inputExtractMapper is not connected at this point - only if/when it is used from within the graph
          CRoxieServerActivity::connectInputStreams(consumerOrdered);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return inputExtractMapper->queryInput();
          else
              return NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerGraphLoopActivity::start(parentExtractSize, parentExtract, paused);         // initialises GraphExtractBuilder
          inputExtractMapper->setParentExtract(parentExtractSize, parentExtract);
  
          createExpandedGraph(GraphExtractBuilder.size(), GraphExtractBuilder.getbytes(), probeManager);
          resultInput->start(GraphExtractBuilder.size(), GraphExtractBuilder.getbytes(), paused);
          startJunction(resultJunction);
      }
  
      virtual void stop()
      {
          if (resultStream)
              resultStream->stop();
          CRoxieServerGraphLoopActivity::stop();
      }
  
      virtual void reset()
      {
          if (resultInput)
              resultInput->reset();
          resetJunction(resultJunction);
          resultInput = NULL;
          resultStream = NULL;
          resultJunction.clear();
  
          ForEachItemIn(i, iterationGraphs)
              iterationGraphs.item(i).gatherStatistics(childStats);
  
          outputs.kill();
          iterationGraphs.kill(); // must be done after all activities killed
          if (probeManager)
          {
              probeManager->deleteGraph(NULL, (IArrayOf<IInputBase>*)&probes);
              probes.kill();
          }
  
          CRoxieServerGraphLoopActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * ret = resultStream->nextRow();
          if (ret)
              processed++;
          return ret;
      }
  
      void createExpandedGraph(unsigned parentExtractSize, const byte *parentExtract, IProbeManager *probeManager)
      {
          //result(0) is the input to the graph.
          resultInput = inputExtractMapper;
          outputs.append(* new CGraphIterationInfo(resultInput->queryActivity(), resultInput, 0, 1));
  
          for (createLoopCounter=1; createLoopCounter <= maxIterations; createLoopCounter++)
          {
              IRoxieServerChildGraph * graph = childGraph->createGraphLoopInstance(ctx, createLoopCounter, parentExtractSize, parentExtract, *this);
              graph->beforeExecute();
              iterationGraphs.append(*graph);
              graph->gatherIterationUsage(*this);
              CGraphIterationInfo *iteration = graph->selectGraphLoopOutput();
              outputs.append(*iteration);
          }
          createLoopCounter = 0;
  
          createSplitters(probeManager, probes);
  
          ForEachItemIn(i2, iterationGraphs)
              iterationGraphs.item(i2).associateIterationOutputs(*this);
          resultInput = outputs.tos().connectOutput(probeManager, probes, this, 0);
          resultStream = connectSingleStream(ctx, resultInput, 0, resultJunction, true);
      }
  
      void createSplitters(IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes)
      {
          ForEachItemIn(i, outputs)
          {
              CGraphIterationInfo & next = outputs.item(i);
              next.createSplitter(ctx, probeManager, probes);
          }
      }
  
  //IRoxieServerLoopResultProcessor
      virtual void noteUseIteration(unsigned _whichIteration)
      {
          int whichIteration = (int) _whichIteration; // May go negative - API is unsigned for historical reasons
          if (whichIteration >= 0)
          {
              if (!outputs.isItem(whichIteration))
                  throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Error reading graph result %d from iteration %d", whichIteration, createLoopCounter);
              outputs.item(whichIteration).noteUsed();
          }
      }
  
      virtual IFinalRoxieInput* connectIterationOutput(unsigned whichIteration, IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes, IRoxieServerActivity *targetAct, unsigned targetIdx)
      {
          if (outputs.isItem(whichIteration))
          {
              CGraphIterationInfo & next = outputs.item(whichIteration);
              return next.connectOutput(probeManager, probes, targetAct, targetIdx);
          }
          return NULL;
      }
  };
  
  //=================================================================================
  
  class CRoxieServerGraphLoopActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned loopGraphId;
      unsigned flags;
  
      Linked<IOutputMetaData> counterMeta;
  
  public:
      CRoxieServerGraphLoopActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _loopGraphId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), loopGraphId(_loopGraphId)
      {
          Owned<IHThorGraphLoopArg> helper = (IHThorGraphLoopArg *) helperFactory();
          flags = helper->getFlags();
          counterMeta.setown(new CCounterRowMetaData);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (kind == TAKparallelgraphloop)
              return new CRoxieServerParallelGraphLoopActivity(_ctx, this, _probeManager, loopGraphId, counterMeta);
          else
              return new CRoxieServerSequentialGraphLoopActivity(_ctx, this, _probeManager, loopGraphId, counterMeta);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerGraphLoopActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _loopGraphId)
  {
      return new CRoxieServerGraphLoopActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _loopGraphId);
  }
  
  //=====================================================================================================
  
  class CRoxieServerLibraryCallActivity : public CRoxieServerActivity
  {
      class OutputAdaptor : public CExtractMapperInput
      {
          bool stopped;
  
      public:
          CRoxieServerLibraryCallActivity *parent;
          IOutputMetaData * meta = nullptr;
          unsigned oid;
          unsigned processed;
          unsigned numStarts = 0;
  
      public:
          OutputAdaptor()
          {
              parent = NULL;
              oid = 0;
              init();
          }
  
          ~OutputAdaptor()
          {
              if (traceStartStop)
                  DBGLOG("%p ~OutputAdaptor %d", this, oid);
          }
  
          void setParent(CRoxieServerLibraryCallActivity * _parent, IOutputMetaData * _meta, unsigned _oid)
          {
              parent = _parent;
              meta = _meta;
              oid = _oid;
          }
  
          void init()
          {
              processed = 0;
              stopped = false;
          }
  
          virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
          {
              parent->start(oid, parentExtractSize, parentExtract, paused);
              CExtractMapperInput::start(parentExtractSize, parentExtract, paused);
              numStarts++;
          }
  
          virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
          {
              //MORE value of consumerOrdered depends on whether this activity is marked as ordered.
              if (!oid)  // Only need to set up once
              {
                  parent->connectInputStreams(true);  // To handle dependencies etc
              }
              return CExtractMapperInput::getOutputStreams(ctx, 0, streams, nullptr, true, nullptr);
          }
  
  
          //This override should not really be needed, but input is set up too late for the call to connect strands
          virtual IOutputMetaData * queryOutputMeta() const
          {
              return meta;
          }
  
          virtual void stop()
          {
              if (!stopped)
              {
                  stopped = true;
                  parent->stop(oid);  // parent code relies on stop being called exactly once per adaptor, so make sure it is!
                  CExtractMapperInput::stop();
              }
          };
  
          virtual void reset()
          {
              parent->reset(oid);
              CExtractMapperInput::reset();
              init();
          };
  
          virtual void checkAbort()
          {
              parent->checkAbort();
          }
      };
      
      IHThorLibraryCallArg &helper;
      unsigned numInputs;
      unsigned numOutputs;
      unsigned numActiveOutputs;
      bool started;
      OutputAdaptor* outputAdaptors;
      CExtractMapperInput * * inputAdaptors;
      bool * inputUsed;
      bool * outputUsed;
      Owned<IException> error;
      CriticalSection crit;
      rtlRowBuilder libraryExtractBuilder;
      Owned<IQueryFactory> libraryQuery;  // Release after libraryGraph...
      Owned<IActivityGraph> libraryGraph;
      const LibraryCallFactoryExtra & extra;
  
  public:
      CRoxieServerLibraryCallActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs, unsigned _numOutputs, const LibraryCallFactoryExtra & _extra)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorLibraryCallArg &)basehelper), extra(_extra)
      {
          numInputs = _numInputs;
          numOutputs = _numOutputs;
          numActiveOutputs = numOutputs;
          inputAdaptors = new CExtractMapperInput*[numInputs];
          inputUsed = new bool[numInputs];
          for (unsigned i1 = 0; i1 < numInputs; i1++)
          {
              inputAdaptors[i1] = new CExtractMapperInput;
              inputUsed[i1] = false;
          }
  
          outputAdaptors = new OutputAdaptor[numOutputs];
          outputUsed = new bool[numOutputs];
          for (unsigned i2 = 0; i2 < numOutputs; i2++)
          {
              unsigned actualOutput = extra.outputs.item(i2);
              outputAdaptors[i2].setParent(this, helper.queryOutputMeta(actualOutput), i2);
              outputUsed[i2] = false;
          }
          started = false;
      }
  
      ~CRoxieServerLibraryCallActivity()
      {
          for (unsigned i1 = 0; i1 < numInputs; i1++)
              ::Release(inputAdaptors[i1]);
          delete [] inputAdaptors;
          delete [] inputUsed;
          delete [] outputAdaptors;
          delete [] outputUsed;
      }
  
      virtual void noteLibrary(IQueryFactory *library) override
      {
          libraryQuery.set(library);
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          libraryGraph.setown(ctx->getLibraryGraph(extra, this));
          libraryGraph->onCreate(_colocalParent);
  
          //Now map the inputs and outputs to the adapters
          IRoxieServerChildGraph * graph = libraryGraph->queryLoopGraph();
          for (unsigned i1=0; i1<numInputs; i1++)
              inputUsed[i1] = graph->querySetInputResult(i1, 0, inputAdaptors[i1]);
  
          for (unsigned i2=0; i2<numOutputs; i2++)
          {
              unsigned outputIndex = extra.outputs.item(i2);
              Owned<IFinalRoxieInput> output = graph->selectOutput(numInputs+outputIndex);
              outputAdaptors[i2].setInput(0, output);   // MORE - no idea if this 0 is right??
              outputAdaptors[i2].connectInputStreams(ctx, true);
          }
      }
  
      virtual void start(unsigned oid, unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CriticalBlock b(crit);
          if (error)
              throw error.getLink();
          if (collectFactoryStatistics)
              factory->noteStarted(oid);
          if (!started)
          {
              // even though it is not complete, we don't want to run this again if it fails.
              started = true;
  
              //see notes on splitter above
              try
              {
                  CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
              }
              catch (IException *E)
              {
  #ifdef TRACE_SPLIT
                  CTXLOG("spill %d caught exception in start", activityId);
  #endif
                  error.set(E);
                  throw;
              }
              catch (...)
              {
                  IException *E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught in CRoxieServerLibraryCallActivity::start");
                  error.set(E);
                  throw E;
              }
  
              //recreate the parent extract, and use it to reinitialize the graphs...
              libraryExtractBuilder.clear();
              helper.createParentExtract(libraryExtractBuilder);
  
              // NOTE - do NOT set activeOutputs = numOutputs here - we must rely on the value set in reset and constructor. This is because we can see stop on
              // some inputs before we see start on others.
              for (unsigned i1 = 0; i1 < numInputs; i1++)
              {
                  if (inputUsed[i1])
                      inputAdaptors[i1]->setParentExtract(parentExtractSize, parentExtract);
                  else
                      inputAdaptors[i1]->queryInput()->stopall();
              }
  
              for (unsigned i2 = 0; i2 < numOutputs; i2++)
                  outputAdaptors[i2].setParentExtract(libraryExtractBuilder.size(), libraryExtractBuilder.getbytes());
  
              //call stop on all the unused inputs.
              IRoxieServerChildGraph * graph = libraryGraph->queryLoopGraph();
              graph->beforeExecute();
              ForEachItemIn(i3, extra.unusedOutputs)
              {
                  Owned<IFinalRoxieInput> output = graph->selectOutput(numInputs+extra.unusedOutputs.item(i3));
                  output->stopall();
              }
          }
      }
  
      virtual void stop(unsigned oid)
      {
          CriticalBlock b(crit);
          if (--numActiveOutputs == 0)
          {
              //call stop on all the unused inputs.
              IRoxieServerChildGraph * graph = libraryGraph->queryLoopGraph();
              graph->beforeExecute();
              ForEachItemIn(i3, extra.unusedOutputs)
              {
                  Owned<IFinalRoxieInput> output = graph->selectOutput(numInputs+extra.unusedOutputs.item(i3));
                  output->stopall();
              }
              CRoxieServerActivity::stop();
          }
      }
  
      void reset(unsigned oid)
      {
          started = false;
          error.clear();
          numActiveOutputs = numOutputs;
          if (state != STATEreset) // make sure input is only reset once
          {
              CRoxieServerActivity::reset();
              libraryGraph->reset();
              //Call reset on all unused inputs/outputs from the graph - no one else will.
              for (unsigned i1 = 0; i1 < numInputs; i1++)
              {
                  if (!inputUsed[i1])
                      inputAdaptors[i1]->reset();
              }
              IRoxieServerChildGraph * graph = libraryGraph->queryLoopGraph();
              ForEachItemIn(i3, extra.unusedOutputs)
              {
                  Owned<IFinalRoxieInput> output = graph->selectOutput(numInputs+extra.unusedOutputs.item(i3));
                  output->reset();
              }
          }
      };
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          inputAdaptors[idx]->setInput(_sourceIdx, _in);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx < numInputs && inputAdaptors[idx])
              return inputAdaptors[idx]->queryInput();
          else
              return NULL;
      }
  
  public:
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // Internal logic error - we are not anybody's input
      }
  
      virtual IOutputMetaData * queryOutputMeta() const
      {
          throwUnexpected();      // should be called on outputs instead
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          assertex(idx!=(unsigned)-1);
          assertex(!outputUsed[idx]);
          outputUsed[idx] = true;
          return &outputAdaptors[idx];
      }
  
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override
      {
          if (libraryGraph)
              libraryGraph->gatherStatistics(statsBuilder);
          CRoxieServerActivity::gatherStatistics(statsBuilder);
      }
  
      virtual void updateEdgeStats(IStatisticGatherer * statsBuilder) const
      {
          for (unsigned i = 0; i < numOutputs; i++)
              addEdgeStats(statsBuilder, i, outputAdaptors[i].numStarts, outputAdaptors[i].processed, 0);
      }
  
  };
  
  
  void LibraryCallFactoryExtra::set(const LibraryCallFactoryExtra & _other)
  {
      ForEachItemIn(i1, _other.outputs)
          outputs.append(_other.outputs.item(i1));
      ForEachItemIn(i2, _other.unusedOutputs)
          unusedOutputs.append(_other.unusedOutputs.item(i2));
      maxOutputs = _other.maxOutputs;
      graphid = _other.graphid;
      libraryName.set(_other.libraryName);
      embeddedGraphName.set(_other.embeddedGraphName);
      interfaceHash = _other.interfaceHash;
      embedded = _other.embedded;
  
  }
  
  void LibraryCallFactoryExtra::calcUnused()
  {
      for (unsigned i=0; i < maxOutputs; i++)
          if (!outputs.contains(i))
              unusedOutputs.append(i);
  }
  
  class CRoxieServerLibraryCallActivityFactory : public CRoxieServerMultiOutputFactory
  {
  private:
      CRoxieServerMultiInputInfo inputs;
      LibraryCallFactoryExtra extra;
  
  public:
      CRoxieServerLibraryCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, LibraryCallFactoryExtra & _extra)
          : CRoxieServerMultiOutputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          extra.set(_extra);
          extra.calcUnused();
          setNumOutputs(extra.outputs.ordinality());
          // Try to resolve library now so that we can detect interface mismatches early. You could argue it should only be a warning - as it is the query
          // will be suspended and will remain suspended even if a suitable library is later published
          if (!extra.embedded && _graphNode.getPropBool("hint[@name='required']/@value", false))
          {
              StringContextLogger logctx;
              Owned<IQueryFactory> libraryQuery = _queryFactory.lookupLibrary(extra.libraryName, extra.interfaceHash, logctx);
          }
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLibraryCallActivity(_ctx, this, _probeManager, numInputs(), numOutputs, extra);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          inputs.set(idx, source, sourceidx);
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          return inputs.get(idx, sourceidx);
      }
  
      virtual unsigned numInputs() const { return inputs.ordinality(); }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          addXrefLibraryInfo(reply, extra.libraryName);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLibraryCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, LibraryCallFactoryExtra & _extra)
  {
      return new CRoxieServerLibraryCallActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _extra);
  }
  
  //=====================================================================================================
  
  // CRoxieServerNWayInputActivity is a multi-input, multi-output activity, where the outputs are a subset of the inputs
  // Used to implement RANGE(<rowset-expression>, [set-of-indices])
  
  class CRoxieServerNWayInputBaseActivity : public CRoxieServerMultiInputBaseActivity
  {
  protected:
      PointerArrayOf<IFinalRoxieInput> selectedInputs;
      PointerArrayOf<IEngineRowStream> selectedStreams;
      PointerArrayOf<IStrandJunction> selectedJunctions;
  
  public:
      CRoxieServerNWayInputBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs)
      {
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 localCycles = activityStats.totalCycles;
          ForEachItemIn(i, selectedInputs)
          {
              localCycles -= selectedInputs.item(i)->queryTotalCycles();
          }
          if (localCycles < 0)
              localCycles = 0;
          return localCycles;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (selectedInputs.isItem(idx))
              return selectedInputs.item(idx);
          else
              return NULL;
      }
  
      virtual void reset()    
      {
          ForEachItemIn(i, selectedInputs)
              selectedInputs.item(i)->reset();
          selectedInputs.kill();
          selectedStreams.kill();
          selectedJunctions.kill();
          CRoxieServerMultiInputBaseActivity::reset(); 
      }
  
  
      virtual const void * nextRow()
      {
          throwUnexpected();
      }
  
      virtual unsigned numConcreteOutputs() const
      {
          return selectedInputs.ordinality();
      }
  
      virtual IFinalRoxieInput * queryConcreteInput(unsigned idx)
      {
          return queryInput(idx);
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
      {
          if (selectedStreams.isItem(whichInput))
              return selectedStreams.item(whichInput);
          return NULL;
      }
  
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
      {
          if (selectedJunctions.isItem(idx))
              return selectedJunctions.item(idx);
          return NULL;
      }
  };
  
  class CRoxieServerNWayInputActivity : public CRoxieServerNWayInputBaseActivity
  {
      IHThorNWayInputArg & helper;
  public:
      CRoxieServerNWayInputActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayInputBaseActivity(_ctx, _factory, _probeManager, _numInputs), helper((IHThorNWayInputArg &)basehelper)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerNWayInputBaseActivity::start(parentExtractSize, parentExtract, paused);
  
          bool selectionIsAll;
          size32_t selectionLen;
          rtlDataAttr selection;
          helper.getInputSelection(selectionIsAll, selectionLen, selection.refdata());
  
          selectedInputs.kill();
          selectedStreams.kill();
          selectedJunctions.kill();
          assertex(numInputs==numStreams); // Will need refactoring when that ceases to be true
          if (selectionIsAll)
          {
              for (unsigned i=0; i < numInputs; i++)
              {
                  selectedInputs.append(inputArray[i]);
                  selectedStreams.append(streamArray[i]);    // Assumes 1:1 relationship - is that good?
                  selectedJunctions.append(junctionArray[i]);
              }
          }
          else
          {
              const size32_t * selections = (const size32_t *)selection.getdata();
              unsigned max = selectionLen/sizeof(size32_t);
              for (unsigned i = 0; i < max; i++)
              {
                  unsigned nextIndex = selections[i];
                  //Check there are no duplicates.....  Assumes there are a fairly small number of inputs, so n^2 search is ok.
                  for (unsigned j=i+1; j < max; j++)
                  {
                      if (nextIndex == selections[j])
                          throw MakeStringException(ROXIE_NWAY_INPUT_ERROR, "Selection list for nway input can not contain duplicates");
                  }
                  if (nextIndex > numInputs || nextIndex > numStreams)
                      throw MakeStringException(ROXIE_NWAY_INPUT_ERROR, "Index %d in RANGE selection list is out of range", nextIndex);
  
                  selectedInputs.append(inputArray[nextIndex-1]);
                  selectedStreams.append(streamArray[nextIndex-1]);    // Assumes 1:1 relationship - is that good?
                  selectedJunctions.append(junctionArray[nextIndex-1]);
              }
          }
          // NB: Whatever pulls this nwayinput activity, starts and stops the selectedInputs and selectedJunctions
          // But the N-Way input itself is now done with processing, and can stop itself/dependencies.
          stop();
      }
  
  };
  
  class CRoxieServerNWayInputActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerNWayInputActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNWayInputActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNWayInputActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNWayInputActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerNWayGraphLoopResultReadActivity : public CRoxieServerNWayInputBaseActivity
  {
      IHThorNWayGraphLoopResultReadArg & helper;
      IArrayOf<IRoxieServerActivity> resultReaders;
      Owned<IStrandJunction> *resultJunctions;
      unsigned graphId;
      bool grouped;
      bool selectionIsAll;
      size32_t selectionLen;
      rtlDataAttr selection;
  
  public:
      CRoxieServerNWayGraphLoopResultReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _graphId)
          : CRoxieServerNWayInputBaseActivity(_ctx, _factory, _probeManager, 0), helper((IHThorNWayGraphLoopResultReadArg &)basehelper)
      {
          grouped = helper.isGrouped();
          graphId = _graphId;
          selectionIsAll = false;
          selectionLen = 0;
          resultJunctions = NULL;
      }
      ~CRoxieServerNWayGraphLoopResultReadActivity()
      {
          delete [] resultJunctions;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
  
          if (selectedInputs.ordinality() == 0)
          {
              initInputSelection();
  
              unsigned max = selectionLen / sizeof(size32_t);
              const size32_t * selections = (const size32_t *)selection.getdata();
              IProbeManager * probeManager = NULL;        // MORE!!
              for (unsigned i = 0; i < max; i++)
              {
                  CRoxieServerActivity * resultInput = new CRoxieServerInternalGraphLoopResultReadActivity(ctx, factory, probeManager, graphId, selections[i]);
                  resultReaders.append(*resultInput);
                  selectedInputs.append(resultInput);
                  resultInput->onCreate(colocalParent);
                  resultInput->start(parentExtractSize, parentExtract, paused);
                  selectedStreams.append(connectSingleStream(ctx, resultInput, 0, resultJunctions[i], true));
                  selectedJunctions.append(resultJunctions[i]);
              }
          }
          // I have done with my processing - note that this won't stop my inputs
          stop();
      }
  
      virtual void reset()    
      {
          resultReaders.kill();
          CRoxieServerNWayInputBaseActivity::reset();
          ForEachItemIn(i, selectedStreams)
              resetJunction(resultJunctions[i]);
      }
  
      virtual void gatherIterationUsage(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract)
      {
          ensureCreated();
          basehelper.onStart(parentExtract, NULL);
          initInputSelection();
  
          unsigned max = selectionLen / sizeof(size32_t);
          const size32_t * selections = (const size32_t *)selection.getdata();
          for (unsigned i = 0; i < max; i++)
              processor.noteUseIteration(selections[i]);
      }
  
      virtual void associateIterationOutputs(IRoxieServerLoopResultProcessor & processor, unsigned parentExtractSize, const byte * parentExtract, IProbeManager *probeManager, IArrayOf<IRoxieProbe> &probes)
      {
          //selection  etc. already initialised from the gatherIterationUsage() call.
          unsigned max = selectionLen / sizeof(size32_t);
          const size32_t * selections = (const size32_t *)selection.getdata();
          resultJunctions = new Owned<IStrandJunction> [max];
          for (unsigned i = 0; i < max; i++)
          {
              IFinalRoxieInput *in = processor.connectIterationOutput(selections[i], probeManager, probes, this, i);
              selectedInputs.append(in);
              selectedStreams.append(connectSingleStream(ctx, in, 0, resultJunctions[i], true));
              selectedJunctions.append(resultJunctions[i]);
          }
      }
  
  protected:
      void initInputSelection()
      {
          helper.getInputSelection(selectionIsAll, selectionLen, selection.refdata());
          if (selectionIsAll)
              throw MakeStringException(ROXIE_NWAY_INPUT_ERROR, "ALL not yet supported for NWay graph inputs");
      }
  };
  
  
  class CRoxieServerNWayGraphLoopResultReadActivityFactory : public CRoxieServerActivityFactory
  {
       unsigned graphId;
  
  public:
      CRoxieServerNWayGraphLoopResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _graphId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), graphId(_graphId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNWayGraphLoopResultReadActivity(_ctx, this, _probeManager, graphId);
      }
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for NWay GraphLoopResultRead activity");
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNWayGraphLoopResultReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned graphId)
  {
      return new CRoxieServerNWayGraphLoopResultReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, graphId);
  }
  
  //=================================================================================
  
  
  class RoxieSteppedInput : implements ISteppedInput, public CInterface
  {
  public:
      RoxieSteppedInput(IFinalRoxieInput * _input, IEngineRowStream *_stream) { input = _input; inputStream = _stream; }
      IMPLEMENT_IINTERFACE
  
  protected:
      virtual const void * nextInputRow()
      {
  #ifdef TRACE_SEEK_REQUESTS
          IRoxieContextLogger * logger = input->queryActivity();
          const void * ret = doNextInputRow();
          {
              CommonXmlWriter xmlwrite(XWFtrim|XWFopt|XWFnoindent);
              if (!ret)
                  xmlwrite.outputBool(true,"eof");
              else if (input->queryOutputMeta()->hasXML())
                  input->queryOutputMeta()->toXML((byte *) ret, xmlwrite);
              logger->CTXLOG("next() returns (%s)", xmlwrite.str());
          }
          return ret;
  #else
          return inputStream->ungroupedNextRow();
  #endif
      }
  
      virtual const void * nextInputRowGE(const void * seek, unsigned numFields, bool & wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
  #ifdef TRACE_SEEK_REQUESTS
          IRoxieContextLogger * logger = input->queryActivity();
          {
              CommonXmlWriter xmlwrite(XWFtrim|XWFopt|XWFnoindent);
              if (input->queryOutputMeta()->hasXML())
                  input->queryOutputMeta()->toXML((byte *) seek, xmlwrite);
              logger->CTXLOG("nextInputRowGE(%d, %s%s%s, %s) seek(%s)", 
                      numFields, 
                      stepExtra.readAheadManyResults() ? "readahead " : "",  
                      stepExtra.returnMismatches() ? "mismatch" : "exact",
                      stepExtra.onlyReturnFirstSeekMatch() ? " single-match" : "",
                      stepExtra.queryExtraSeeks() ? "multi-seek":"", 
                      xmlwrite.str());
          }
          const void * ret = doNextInputRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          {
              CommonXmlWriter xmlwrite(XWFtrim|XWFopt|XWFnoindent);
              if (!ret)
                  xmlwrite.outputBool(true,"eof");
              else if (input->queryOutputMeta()->hasXML())
                  input->queryOutputMeta()->toXML((byte *) ret, xmlwrite);
              logger->CTXLOG("nextInputRowGE(%d, %s%s%s, %s) result(%s)", 
                      numFields, 
                      stepExtra.readAheadManyResults() ? "readahead " : "",  
                      stepExtra.returnMismatches() ? "mismatch" : "exact",
                      stepExtra.onlyReturnFirstSeekMatch() ? " single-match" : "",
                      stepExtra.queryExtraSeeks() ? "multi-seek":"", 
                      xmlwrite.str());
          }
          return ret;
  #else
          return doNextInputRowGE(seek, numFields, wasCompleteMatch, stepExtra);
  #endif
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          inputStream->resetEOF();
      }
  
      virtual IInputSteppingMeta * queryInputSteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
      inline const void * doNextInputRowGE(const void * seek, unsigned numFields, bool & wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          assertex(wasCompleteMatch);
          return inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
      }
  
  
  protected:
      IFinalRoxieInput * input;
      IEngineRowStream * inputStream;
  };
  
  //=================================================================================
  
  class CRoxieServerNWayBaseActivity : public CRoxieServerMultiInputBaseActivity
  {
  public:
      CRoxieServerNWayBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs)
      {
      }
  
      void startExpandedInputs(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          ForEachItemIn(ei, expandedInputs)
              expandedInputs.item(ei)->start(parentExtractSize, parentExtract, paused);
          ForEachItemIn(idx, expandedInputs)
              startJunction(expandedJunctions.item(idx));
      }
  
      virtual void stop()
      {
          ForEachItemIn(i, expandedStreams)
              expandedStreams.item(i)->stop();
          CRoxieServerMultiInputBaseActivity::stop();
      }
  
      virtual void reset()
      {
          ForEachItemIn(idx, expandedInputs)
              resetJunction(expandedJunctions.item(idx));
          expandedInputs.kill();
          expandedStreams.kill();
          expandedJunctions.kill();
          CRoxieServerMultiInputBaseActivity::reset();
      }
  protected:
      PointerArrayOf<IFinalRoxieInput> expandedInputs;
      PointerArrayOf<IEngineRowStream> expandedStreams;
      PointerArrayOf<IStrandJunction> expandedJunctions;
  };
  
  //=================================================================================
  
  class CRoxieServerNaryActivity : public CRoxieServerNWayBaseActivity
  {
  public:
      CRoxieServerNaryActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayBaseActivity(_ctx, _factory, _probeManager, _numInputs)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerNWayBaseActivity::start(parentExtractSize, parentExtract, paused);
  
          for (unsigned i=0; i < numInputs; i++)
          {
              IFinalRoxieInput * cur = inputArray[i];
              CRoxieServerNWayInputBaseActivity *nWayInput = dynamic_cast<CRoxieServerNWayInputBaseActivity *>(cur);
              if (nWayInput)
              {
                  nWayInput->start(parentExtractSize, parentExtract, paused);
                  unsigned numRealInputs = cur->numConcreteOutputs();
                  for (unsigned j = 0; j < numRealInputs; j++)
                  {
                      expandedInputs.append(cur->queryConcreteInput(j));
                      expandedStreams.append(cur->queryConcreteOutputStream(j));
                      expandedJunctions.append(cur->queryConcreteOutputJunction(j));
                  }
              }
              else
              {
                  expandedInputs.append(cur);
                  // NB: this activities input streams + junction have been setup and held in CRoxieServerMultiInputActivity base
                  expandedStreams.append(queryConcreteOutputStream(i));
                  expandedJunctions.append(queryConcreteOutputJunction(i));
              }
          }
          startExpandedInputs(parentExtractSize, parentExtract, paused);
      }
  };
  
  //=================================================================================
  
  class CRoxieStreamMerger : public CStreamMerger
  {
  public:
      CRoxieStreamMerger() : CStreamMerger(true)
      {
          streamArray = NULL;
      }
  
      void initInputs(unsigned _numInputs, IEngineRowStream ** _streamArray)
      {
          CStreamMerger::initInputs(_numInputs);
          streamArray = _streamArray;
      }
  
      virtual bool pullInput(unsigned i, const void * seek, unsigned numFields, const SmartStepExtra * stepExtra)
      {
          const void * next;
          bool matches = true;
          if (seek)
              next = streamArray[i]->nextRowGE(seek, numFields, matches, *stepExtra);
          else
              next = streamArray[i]->ungroupedNextRow();
          pending[i] = next;
          pendingMatches[i] = matches;
          return (next != NULL);
      }
  
      virtual void releaseRow(const void * row)
      {
          ReleaseRoxieRow(row);
      }
  
  protected:
      IEngineRowStream **streamArray;
  };
  
  
  class CRoxieServerNWayMergeActivity : public CRoxieServerNaryActivity
  {
  public:
      CRoxieServerNWayMergeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNaryActivity(_ctx, _factory, _probeManager, _numInputs),
          helper((IHThorNWayMergeArg &)basehelper) 
      {
          initializedMeta = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerNaryActivity::start(parentExtractSize, parentExtract, paused);
          merger.init(helper.queryCompare(), helper.dedup(), helper.querySteppingMeta()->queryCompare());
          merger.initInputs(expandedStreams.length(), expandedStreams.getArray());
      }
  
      virtual void stop()
      {
          merger.done();
          CRoxieServerNaryActivity::stop();
      }
  
      virtual void reset()    
      {
          merger.cleanup();
          CRoxieServerNaryActivity::reset(); 
          initializedMeta = false;
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = merger.nextRow();
          if (next)
              processed++;
          return next;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool & wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = merger.nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (next)
              processed++;
          return next;
      }
  
      virtual IInputSteppingMeta * querySteppingMeta()
      {
          if (expandedStreams.ordinality() == 0)
              return NULL;
          if (!initializedMeta)
          {
              meta.init(helper.querySteppingMeta(), false);
              ForEachItemIn(i, expandedStreams)
              {
                  if (meta.getNumFields() == 0)
                      break;
                  IInputSteppingMeta * inputMeta = expandedInputs.item(i)->querySteppingMeta();
                  meta.intersect(inputMeta);
              }
              initializedMeta = true;
          }
          if (meta.getNumFields() == 0)
              return NULL;
          return &meta;
      }
  
  protected:
      IHThorNWayMergeArg &helper;
      CRoxieStreamMerger merger;
      CSteppingMeta meta;
      bool initializedMeta;
  };
  
  
  class CRoxieServerNWayMergeActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerNWayMergeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNWayMergeActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNWayMergeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNWayMergeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerNWayMergeJoinActivity : public CRoxieServerNaryActivity
  {
  public:
      CRoxieServerNWayMergeJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs, CMergeJoinProcessor & _processor)
          : CRoxieServerNaryActivity(_ctx, _factory, _probeManager, _numInputs),
          helper((IHThorNWayMergeJoinArg &)basehelper),
          processor(_processor)
      {
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerNaryActivity::onCreate(_colocalParent);
          inputAllocator.setown(createRowAllocator(helper.queryInputMeta()));
          outputAllocator.setown(createRowAllocator(helper.queryOutputMeta()));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerNaryActivity::start(parentExtractSize, parentExtract, paused);
          ForEachItemIn(i1, expandedInputs)
          {
              Owned<RoxieSteppedInput> stepInput = new RoxieSteppedInput(expandedInputs.item(i1), expandedStreams.item(i1));
              processor.addInput(stepInput);
          }
  
          processor.beforeProcessing(inputAllocator, outputAllocator);
      }
  
      virtual void stop()
      {
          processor.afterProcessing();
          CRoxieServerNaryActivity::stop();
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
      {
          return processor.gatherConjunctions(collector);
      }
  
      virtual void resetEOF() 
      { 
          processor.queryResetEOF(); 
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = processor.nextRow();
          if (next)
              processed++;
          return next;
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * next = processor.nextGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (next)
              processed++;
          return next;
      }
  
      virtual IInputSteppingMeta * querySteppingMeta()
      {
          return processor.queryInputSteppingMeta();
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerNaryActivity::gatherStats(merged);
          if (inputAllocator)
              inputAllocator->gatherStats(merged);
          if (outputAllocator)
              outputAllocator->gatherStats(merged);
      }
  
  protected:
      IHThorNWayMergeJoinArg & helper;
      CMergeJoinProcessor & processor;
      Owned<IEngineRowAllocator> inputAllocator;
      Owned<IEngineRowAllocator> outputAllocator;
  };
  
  
  class CRoxieServerAndMergeJoinActivity : public CRoxieServerNWayMergeJoinActivity
  {
  public:
      CRoxieServerAndMergeJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayMergeJoinActivity(_ctx, _factory, _probeManager, _numInputs, andProcessor), andProcessor(helper)
      {
      }
  
  protected:
      CAndMergeJoinProcessor andProcessor;
  };
  
  class CRoxieServerAndLeftMergeJoinActivity : public CRoxieServerNWayMergeJoinActivity
  {
  public:
      CRoxieServerAndLeftMergeJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayMergeJoinActivity(_ctx, _factory, _probeManager, _numInputs, andLeftProcessor), andLeftProcessor(helper)
      {
      }
  
  protected:
      CAndLeftMergeJoinProcessor andLeftProcessor;
  };
  
  class CRoxieServerMofNMergeJoinActivity : public CRoxieServerNWayMergeJoinActivity
  {
  public:
      CRoxieServerMofNMergeJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayMergeJoinActivity(_ctx, _factory, _probeManager, _numInputs, mofnProcessor), mofnProcessor(helper)
      {
      }
  
  protected:
      CMofNMergeJoinProcessor mofnProcessor;
  };
  
  class CRoxieServerProximityJoinActivity : public CRoxieServerNWayMergeJoinActivity
  {
  public:
      CRoxieServerProximityJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayMergeJoinActivity(_ctx, _factory, _probeManager, _numInputs, proximityProcessor), proximityProcessor(helper)
      {
      }
  
  protected:
      CProximityJoinProcessor proximityProcessor;
  };
  
  class CRoxieServerNWayMergeJoinActivityFactory : public CRoxieServerMultiInputFactory
  {
      unsigned flags;
  public:
      CRoxieServerNWayMergeJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          Owned<IHThorNWayMergeJoinArg> helper = (IHThorNWayMergeJoinArg *) helperFactory();
          flags = helper->getJoinFlags();
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (flags & IHThorNWayMergeJoinArg::MJFhasrange)
              return new CRoxieServerProximityJoinActivity(_ctx, this, _probeManager, numInputs());
  
          switch (flags & IHThorNWayMergeJoinArg::MJFkindmask)
          {
          case IHThorNWayMergeJoinArg::MJFinner:
              return new CRoxieServerAndMergeJoinActivity(_ctx, this, _probeManager, numInputs());
          case IHThorNWayMergeJoinArg::MJFleftonly:
          case IHThorNWayMergeJoinArg::MJFleftouter:
              return new CRoxieServerAndLeftMergeJoinActivity(_ctx, this, _probeManager, numInputs());
          case IHThorNWayMergeJoinArg::MJFmofn:
              return new CRoxieServerMofNMergeJoinActivity(_ctx, this, _probeManager, numInputs());
          default:
              throwUnexpected();
          }
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNWayMergeJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNWayMergeJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  
  //=================================================================================
  
  class CRoxieServerNWaySelectActivity : public CRoxieServerNWayBaseActivity
  {
      IHThorNWaySelectArg &helper;
  public:
      CRoxieServerNWaySelectActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerNWayBaseActivity(_ctx, _factory, _probeManager, _numInputs),
            helper((IHThorNWaySelectArg &)basehelper)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerNWayBaseActivity::start(parentExtractSize, parentExtract, paused);
  
          unsigned whichInput = helper.getInputIndex();
          selectedInput = nullptr;
          selectedStream = nullptr;
          if (whichInput--)
          {
              for (unsigned i=0; i < numInputs; i++)
              {
                  IFinalRoxieInput * cur = inputArray[i];
                  CRoxieServerNWayInputBaseActivity *nWayInput = dynamic_cast<CRoxieServerNWayInputBaseActivity *>(cur);
                  if (nWayInput)
                  {
                      nWayInput->start(parentExtractSize, parentExtract, paused);
                      unsigned numRealInputs = cur->numConcreteOutputs();
                      if (whichInput < numRealInputs)
                      {
                          expandedInputs.append(cur->queryConcreteInput(whichInput));
                          expandedStreams.append(cur->queryConcreteOutputStream(whichInput));
                          expandedJunctions.append(cur->queryConcreteOutputJunction(whichInput));
                          break;
                      }
                      whichInput -= numRealInputs;
                  }
                  else
                  {
                      if (whichInput == 0)
                      {
                          expandedInputs.append(cur);
                          // NB: this activities input streams + junction have been setup and held in CRoxieServerMultiInputActivity base
                          expandedStreams.append(queryConcreteOutputStream(i));
                          expandedJunctions.append(queryConcreteOutputJunction(i));
                          break;
                      }
                      whichInput -= 1;
                  }
              }
          }
          if (expandedInputs.ordinality())
          {
              startExpandedInputs(parentExtractSize, parentExtract, paused);
              selectedInput = expandedInputs.item(0);
              selectedStream = expandedStreams.item(0);
          }
      }
  
      const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!selectedStream)
              return NULL;
          return selectedStream->nextRow();
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          if (!selectedInput)
              return false;
          return selectedInput->gatherConjunctions(collector); 
      }
      
      virtual void resetEOF() 
      { 
          if (selectedStream)
              selectedStream->resetEOF();
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!selectedStream)
              return NULL;
          return selectedStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          if (selectedInput)
              return selectedInput->querySteppingMeta();
          return NULL;
      }
  
  protected:
      IFinalRoxieInput * selectedInput = nullptr;
      IEngineRowStream * selectedStream = nullptr;
      Owned<IStrandJunction> selectedJunction;
  };
  
  
  class CRoxieServerNWaySelectActivityFactory : public CRoxieServerMultiInputFactory
  {
  public:
      CRoxieServerNWaySelectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerNWaySelectActivity(_ctx, this, _probeManager, numInputs());
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerNWaySelectActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerNWaySelectActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerRemoteActivity : public CRoxieServerActivity, implements IRoxieServerErrorHandler
  {
  protected:
      IHThorRemoteArg &helper;
      CRemoteResultAdaptor remote;
  
  public:
      CRoxieServerRemoteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteID)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            helper((IHThorRemoteArg &)basehelper),
            remote(_ctx, this, _remoteID, meta.queryOriginal(), helper, *this, false, false) // MORE - if they need it stable we'll have to think!
      {
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return NULL;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          remote.onCreate(_colocalParent);
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          remote.onStart(parentExtractSize, parentExtract);
  
          remote.setLimits(helper.getRowLimit(), (unsigned __int64) -1, I64C(0x7FFFFFFFFFFFFFFF));
          unsigned fileNo = 0;        // MORE - superfiles require us to do this per file part... maybe (needs thought)
          remote.getMem(0, fileNo, 0);  // the cached context is all we need to send
  
          remote.flush();
          remote.senddone();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL: &remote;
      }
  
      virtual void reset()
      {
          processed = remote.processed;
          remote.processed = 0;
          CRoxieServerActivity::reset();
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          helper.onLimitExceeded();
      }
  
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          UNIMPLEMENTED; // MORE - is there an ONFAIL for a limit folded into a remote?
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          remote.gatherStats(merged);
      }
  };
  
  class CRoxieServerRemoteActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      RemoteActivityId remoteId;
      bool isRoot;
  
      CRoxieServerRemoteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, allStatistics), remoteId(_remoteId), isRoot(_isRoot)
      {
          // Note - sets stats to allStatistics (because of potential child queries having stats) but I'm not convinced that is right. Very many activities can have child queries...
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerRemoteActivity(_ctx, this, _probeManager, remoteId);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for %s activity", getActivityText(kind));
      }
  
      virtual bool isSink() const
      {
          //I don't think the action version of this is implemented - but this would be the code
          return isRoot && !meta.queryOriginal();
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerRemoteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId, bool _isRoot)
  {
      return new CRoxieServerRemoteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerIterateActivity : public CRoxieServerActivity
  {
      IHThorIterateArg &helper;
      OwnedConstRoxieRow defaultRecord;
      OwnedConstRoxieRow left;
      OwnedConstRoxieRow right;
      unsigned counter;
  
  public:
      CRoxieServerIterateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            helper((IHThorIterateArg &)basehelper)
      {
          counter = 0;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          counter = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t thisSize = helper.createDefault(rowBuilder);
          defaultRecord.setown(rowBuilder.finalizeRowClear(thisSize));
      }
  
      virtual void reset()    
      {
          defaultRecord.clear();
          right.clear();
          left.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              right.setown(inputStream->nextRow());
              if (!right)
              {
                  bool skippedGroup = (left == NULL) && (counter > 0); //we have just skipped entire group, but shouldn't output a double null
                  left.clear();
                  counter = 0;
                  if (skippedGroup) continue;
                  return NULL;
              }
              try
              {
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  unsigned outSize = helper.transform(rowBuilder, left ? left : defaultRecord, right, ++counter);
                  if (outSize)
                  {
                      left.setown(rowBuilder.finalizeRowClear(outSize));
                      processed++;
                      return left.getLink();
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
          }
      }
  };
  
  class CRoxieServerIterateActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerIterateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerIterateActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIterateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerIterateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerProcessActivity : public CRoxieServerActivity
  {
      IHThorProcessArg &helper;
      OwnedConstRoxieRow curRight;
      OwnedConstRoxieRow initialRight;
      unsigned counter;
      Owned<IEngineRowAllocator> rightRowAllocator;
  
  public:
      CRoxieServerProcessActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorProcessArg &)basehelper)
      {
          counter = 0;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          rightRowAllocator.setown(createRowAllocator(helper.queryRightRecordSize()));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          counter = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
  
          RtlDynamicRowBuilder rowBuilder(rightRowAllocator);
          size32_t thisSize = helper.createInitialRight(rowBuilder);
          initialRight.setown(rowBuilder.finalizeRowClear(thisSize));
          curRight.set(initialRight);
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
  
          try
          {
              for (;;)
              {
                  const void * in = inputStream->nextRow();
                  if (!in)
                  {
                      bool eog = (curRight != initialRight);          // processed any records?
                      counter = 0;
                      curRight.set(initialRight);
                      if (eog)
                          return NULL;
  
                      in = inputStream->nextRow();
                      if (!in)
                          return NULL;
                  }
  
                  RtlDynamicRowBuilder rowBuilder(rowAllocator);
                  RtlDynamicRowBuilder rightRowBuilder(rightRowAllocator);
                  size32_t outSize = helper.transform(rowBuilder, rightRowBuilder, in, curRight, ++counter);
                  ReleaseRoxieRow(in);
  
                  if (outSize)
                  {
                      //MORE: This should be returned...
                      size32_t rightSize = rightRowAllocator->queryOutputMeta()->getRecordSize(rightRowBuilder.getSelf());
                      curRight.setown(rightRowBuilder.finalizeRowClear(rightSize));
                      processed++;
                      return rowBuilder.finalizeRowClear(outSize);
                  }
              }
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          if (rightRowAllocator)
              rightRowAllocator->gatherStats(merged);
      }
  };
  
  class CRoxieServerProcessActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerProcessActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerProcessActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerProcessActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerProcessActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerGroupActivity : public CRoxieServerActivity
  {
      IHThorGroupArg &helper;
      bool endPending;
      bool eof;
      bool first;
      unsigned numGroups;
      unsigned numGroupMax;
      unsigned numProcessedLastGroup;
      const void *next;
  
  public:
      CRoxieServerGroupActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorGroupArg &)basehelper)
      {
          next = NULL;
          endPending = false;
          eof = false;
          first = true;
          numGroups = 0;
          numGroupMax = 0;
          numProcessedLastGroup = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          endPending = false;
          eof = false;
          first = true;
          numGroups = 0;
          numGroupMax = 0;
          numProcessedLastGroup = processed;
          assertex(next == NULL);
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()    
      { 
          noteStatistic(StNumGroups, numGroups);
          noteStatistic(StNumGroupMax, numGroupMax);
          ReleaseClearRoxieRow(next);
          CRoxieServerActivity::reset();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (first)
          {
              next = inputStream->nextRow();
              first = false;
          }
          if (eof || endPending)
          {
              endPending = false;
              return NULL;
          }
  
          const void * prev = next;
          next = inputStream->ungroupedNextRow();
  
          if (next)
          {
              assertex(prev);
              if (!helper.isSameGroup(prev, next))
              {
                  noteEndOfGroup();
                  endPending = true;
              }
          }
          else
          {
              noteEndOfGroup();
              eof = true;
          }
          if (prev)
              processed++;
          return prev;
      }
      inline void noteEndOfGroup()
      {
          unsigned numThisGroup = processed - numProcessedLastGroup;
          if (numThisGroup == 0)
              return;
          numProcessedLastGroup = processed;
          if (numThisGroup > numGroupMax)
              numGroupMax = numThisGroup;
          numGroups++;
      }
  };
  
  class CRoxieServerGroupActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, groupStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerGroupActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerGroupActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerGroupActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerFirstNActivity : public CRoxieServerLateStartActivity
  {
      unsigned __int64 limit;
      unsigned __int64 skip;
      unsigned doneThisGroup;
      IHThorFirstNArg &helper;
  
  public:
      CRoxieServerFirstNActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerLateStartActivity(_ctx, _factory, _probeManager), helper((IHThorFirstNArg &)basehelper)
      {
          doneThisGroup = 0;
          limit = 0;
          skip = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          doneThisGroup = 0;
          CRoxieServerLateStartActivity::start(parentExtractSize, parentExtract, paused);
          limit = helper.getLimit();
          skip = helper.numToSkip();
          lateStart(parentExtractSize, parentExtract, limit > 0);
          if (limit + skip >= limit)
              limit += skip;
      }
  
      const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          const void *ret;
          for (;;)
          {
              ret = inputStream->nextRow();
              if (!ret)
              {
                  if (meta.isGrouped())
                  {
                      if (doneThisGroup > skip)
                      {
                          doneThisGroup = 0;
                          return NULL;
                      }
                      doneThisGroup = 0;
                  }
                  ret = inputStream->nextRow();
                  if (!ret)
                  {
                      eof = true;
                      return NULL;
                  }
              }
              doneThisGroup++;
              if (doneThisGroup > skip)
                  break;
              ReleaseRoxieRow(ret);
          }
  
          if (doneThisGroup <= limit)
          {
              processed++;
              return ret;
          }
  
          ReleaseRoxieRow(ret);
          if (meta.isGrouped())
          {
              while ((ret = inputStream->nextRow()) != NULL)
                  ReleaseRoxieRow(ret);
              doneThisGroup = 0;
          }
          else
              eof = true;
          return NULL;
      }
  };
  
  class CRoxieServerFirstNActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerFirstNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerFirstNActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerFirstNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerFirstNActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSelectNActivity : public CRoxieServerActivity
  {
      bool done;
      IHThorSelectNArg &helper;
  
  public:
      CRoxieServerSelectNActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSelectNArg &)basehelper)
      {
          done = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      const void *defaultRow()
      {
          if (!rowAllocator)
              ensureRowAllocator();      // We delay as often not needed...
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t thisSize = helper.createDefault(rowBuilder);
          return rowBuilder.finalizeRowClear(thisSize);
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done)
              return NULL;
          done = true;
          processed++; // always going to return a row!
          unsigned __int64 index = helper.getRowToSelect();
          while (--index)
          {
              const void * next = inputStream->ungroupedNextRow();
              if (!next)
                  return defaultRow();
              ReleaseRoxieRow(next);
          }
  
          const void * next = inputStream->ungroupedNextRow();
          if (!next)
              next = defaultRow();
  
          return next;
      }
  };
  
  class CRoxieServerSelectNActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSelectNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSelectNActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSelectNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSelectNActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerSelfJoinActivity : public CRoxieServerActivity
  {
      IHThorJoinArg &helper;
      ICompare *collate;
      OwnedRowArray group;
      bool matchedLeft;
      BoolArray matchedRight;
      bool eof;
      bool first;
      unsigned leftIndex;
      unsigned rightIndex;
      unsigned rightOuterIndex;
      unsigned joinLimit;
      unsigned atmostLimit;
      unsigned atmostsTriggered;
      unsigned abortLimit;
      unsigned keepLimit;
      unsigned joinCounter;
      bool leftOuterJoin;
      bool rightOuterJoin;
      bool exclude;
      bool limitFail;
      bool limitOnFail;
      bool cloneLeft;
      bool forceSpill;
      OwnedConstRoxieRow defaultLeft;
      OwnedConstRoxieRow defaultRight;
      OwnedConstRoxieRow lhs;
      Owned<IException> failingLimit;
      bool failingOuterAtmost;
      Owned<IEngineRowAllocator> defaultAllocator;
      Owned<IRHLimitedCompareHelper> limitedhelper;
      Owned<CRHDualCache> dualcache;
      Owned<IGroupedInput> groupedInput;
      IRowStream *dualCacheInput;
  
      bool fillGroup()
      {
          group.clear();
          matchedLeft = false;
          matchedRight.kill();
          failingOuterAtmost = false;
          const void * next;
          unsigned groupCount = 0;
          while((next = groupedInput->nextRow()) != NULL)
          {
              if(groupCount==abortLimit)
              {
                  if(limitFail)
                      failLimit(next);
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  if(limitOnFail)
                  {
                      assertex(!failingLimit);
                      try
                      {
                          failLimit(next);
                      }
                      catch(IException * except)
                      {
                          failingLimit.setown(except);
                      }
                      assertex(failingLimit != NULL);
                      group.append(next);
                      groupCount++;
                      break;
                  }
                  group.clear();
                  groupCount = 0;
                  while(next) 
                  {
                      ReleaseRoxieRow(next);
                      next = groupedInput->nextRow();
                  }
              }
              else if(groupCount==atmostLimit)
              {
                  atmostsTriggered++;
                  if(leftOuterJoin)
                  {
                      group.append(next);
                      groupCount++;
                      failingOuterAtmost = true;
                      break;
                  }
                  else
                  {
                      group.clear();
                      groupCount = 0;
                      while (next) 
                      {
                          ReleaseRoxieRow(next);
                          next = groupedInput->nextRow();
                      }
                  }
              }
              else
              {
                  group.append(next);
                  groupCount++;
              }
          }
          if(group.ordinality()==0)
          {
              eof = true;
              return false;
          }
          leftIndex = 0;
          rightIndex = 0;
          rightOuterIndex = 0;
          joinCounter = 0;
          joinLimit = keepLimit;
          ForEachItemIn(idx, group)
              matchedRight.append(false);
          return true;
      }
  
      void failLimit(const void * next)
      {
          helper.onMatchAbortLimitExceeded();
          CommonXmlWriter xmlwrite(XWFtrim|XWFopt );
          if (!ctx->isBlind() && input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
          {
              input->queryOutputMeta()->toXML((byte *) next, xmlwrite);
          }
          throw MakeStringException(ROXIE_TOO_MANY_RESULTS, "More than %d match candidates in self-join %d for row %s", abortLimit, queryId(), xmlwrite.str());
      }
  
      virtual bool needsAllocator() const { return true; }
  
      const void *joinRecords(const void * curLeft, const void * curRight, unsigned counter, IException * except, unsigned flags)
      {
          try
          {
              if (cloneLeft && !except)
              {
                  LinkRoxieRow(curLeft);
                  return curLeft;
              }
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              size32_t outsize = except ? helper.onFailTransform(rowBuilder, curLeft, curRight, except, flags) : helper.transform(rowBuilder, curLeft, curRight, counter, flags);
              if (outsize)
                  return rowBuilder.finalizeRowClear(outsize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      void createDefaultLeft()
      {
          if (!defaultLeft)
          {
              if (!defaultAllocator)
                  defaultAllocator.setown(createRowAllocator(input->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultAllocator);
              size32_t thisSize = helper.createDefaultLeft(rowBuilder);
              defaultLeft.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
      void createDefaultRight()
      {
          if (!defaultRight)
          {
              if (!defaultAllocator)
                  defaultAllocator.setown(createRowAllocator(input->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultAllocator);
              size32_t thisSize = helper.createDefaultRight(rowBuilder);
              defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
  public:
      CRoxieServerSelfJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _forceSpill)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorJoinArg &)basehelper), forceSpill(_forceSpill)
      {
          collate = helper.queryCompareLeftRight();
          eof = false;
          first = true;
          keepLimit = 0;
          atmostLimit = 0;
          atmostsTriggered = 0;
          unsigned joinFlags = helper.getJoinFlags();
          leftOuterJoin = (joinFlags & JFleftouter) != 0;
          rightOuterJoin = (joinFlags & JFrightouter) != 0;
          cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
          exclude = (joinFlags & JFexclude) != 0;
          abortLimit = 0;
          joinLimit = 0;
          assertex((joinFlags & (JFfirst | JFfirstleft | JFfirstright)) == 0); // no longer supported
          getLimitType(joinFlags, limitFail, limitOnFail);
          if((joinFlags & JFslidingmatch) != 0)
              throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Internal Error: Sliding self join not supported");
          failingOuterAtmost = false;
          matchedLeft = false;
          leftIndex = 0;
          rightIndex = 0;
          rightOuterIndex = 0;
          joinCounter = 0;
          dualCacheInput = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          first = true;
          failingLimit.clear();
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          keepLimit = helper.getKeepLimit();
          if(keepLimit == 0)
              keepLimit = (unsigned)-1;
          atmostsTriggered = 0;
          atmostLimit = helper.getJoinLimit();
          if(atmostLimit == 0)
              atmostLimit = (unsigned)-1;
          else
              assertex(!rightOuterJoin);
          abortLimit = helper.getMatchAbortLimit();
          if (abortLimit == 0) 
              abortLimit = (unsigned)-1;
          if (rightOuterJoin)
              createDefaultLeft();
          if (leftOuterJoin || limitOnFail)
              createDefaultRight();
          ICompare *compareLeft = helper.queryCompareLeft();
          if (helper.isLeftAlreadySorted())
              groupedInput.setown(createGroupedInputReader(inputStream, compareLeft));
          else
          {
              bool isStable = (helper.getJoinFlags() & JFunstable) == 0;
              RoxieSortAlgorithm sortAlgorithm;
              if (forceSpill)
                  sortAlgorithm = isStable ? stableSpillingQuickSortAlgorithm : spillingQuickSortAlgorithm;
              else
                  sortAlgorithm = isStable ? stableQuickSortAlgorithm : quickSortAlgorithm;
              groupedInput.setown(createSortedGroupedInputReader(inputStream, compareLeft, createSortAlgorithm(sortAlgorithm, compareLeft, ctx->queryRowManager(), input->queryOutputMeta(), ctx->queryCodeContext(), tempDirectory, activityId)));
          }
          if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) 
          {   //limited match join (s[1..n])
              dualcache.setown(new CRHDualCache());
              dualcache->init(groupedInput);
              dualCacheInput = dualcache->queryOut1();
              failingOuterAtmost = false;
              matchedLeft = false;
              leftIndex = 0;
              rightOuterIndex = 0;
              joinCounter = 0;
  
              limitedhelper.setown(createRHLimitedCompareHelper());
              limitedhelper->init( helper.getJoinLimit(), dualcache->queryOut2(), collate, helper.queryPrefixCompare() );
          }
      }
  
      virtual void reset()
      {
          if (atmostsTriggered)
              noteStatistic(StNumAtmostTriggered, atmostsTriggered);
          group.clear();
          groupedInput.clear();
          CRoxieServerActivity::reset();
          defaultLeft.clear();
          defaultRight.clear();
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (limitedhelper)
          {
              while(!eof) //limited match join
              {
                  if (!group.isItem(rightIndex))
                  {
                      lhs.setown(dualCacheInput->nextRow());
                      if (lhs)
                      {
                          rightIndex = 0;
                          joinCounter = 0;
                          group.clear();
                          limitedhelper->getGroup(group,lhs);
                      }
                      else 
                      {
                          eof = true;
                      }
                  }
  
                  if (group.isItem(rightIndex))
                  {
                      const void * rhs = group.item(rightIndex++);
                      if(helper.match(lhs, rhs))
                      {
                          const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL, JTFmatchedleft|JTFmatchedright);
                          if(ret)
                          {
                              processed++;
                              return ret;
                          }
                      }
                  }
              }
              return NULL;
          }
          else
          {
              if (first)
              {
                  first = false;
                  fillGroup();
              }
              while(!eof)
              {
                  if(failingOuterAtmost)
                      while(group.isItem(leftIndex))
                      {
                          const void * ret = joinRecords(group.item(leftIndex++), defaultRight, 0, NULL, JTFmatchedleft);
                          if(ret)
                          {
                              processed++;
                              return ret;
                          }
                      }
                  if((joinLimit == 0) || !group.isItem(rightIndex))
                  {
                      if(leftOuterJoin && !matchedLeft && !failingLimit)
                      {
                          const void * ret = joinRecords(group.item(leftIndex), defaultRight, 0, NULL, JTFmatchedleft);
                          if(ret)
                          {
                              matchedLeft = true;
                              processed++;
                              return ret;
                          }
                      }
                      leftIndex++;
                      matchedLeft = false;
                      rightIndex = 0;
                      joinCounter = 0;
                      joinLimit = keepLimit;
                  }
                  if(!group.isItem(leftIndex))
                  {
                      if(failingLimit || failingOuterAtmost)
                      {
                          const void * lhs;
                          while((lhs = groupedInput->nextRow()) != NULL)  // dualCache never active here
                          {
                              const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit, JTFmatchedleft);
                              ReleaseRoxieRow(lhs);
                              if(ret)
                              {
                                  processed++;
                                  return ret;
                              }
                          }
                          failingLimit.clear();
                      }
                      if(rightOuterJoin && !failingLimit)
                          while(group.isItem(rightOuterIndex))
                              if(!matchedRight.item(rightOuterIndex++))
                              {
                                  const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1), 0, NULL, JTFmatchedright);
                                  if(ret)
                                  {
                                      processed++;
                                      return ret;
                                  }
                              }
                      if(!fillGroup())
                          return NULL;
                      continue;
                  }
                  const void * lhs = group.item(leftIndex);
                  if(failingLimit)
                  {
                      leftIndex++;
                      const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit, JTFmatchedleft);
                      if(ret)
                      {
                          processed++;
                          return ret;
                      }
                  }
                  else
                  {
                      const void * rhs = group.item(rightIndex++);
                      if(helper.match(lhs, rhs))
                      {
                          matchedLeft = true;
                          matchedRight.replace(true, rightIndex-1);
                          if(!exclude)
                          {
                              const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL, JTFmatchedleft|JTFmatchedright);
                              if(ret)
                              {
                                  processed++;
                                  joinLimit--;
                                  return ret;
                              }
                          }
                      }
                  }
              }
              return NULL;
          }
      }
  };
  
  class CRoxieServerSelfJoinActivityFactory : public CRoxieServerActivityFactory
  {
      bool forceSpill;
  public:
      CRoxieServerSelfJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, joinStatistics)
      {
          forceSpill = queryFactory.queryOptions().allSortsMaySpill || _graphNode.getPropBool("hint[@name='spill']/@value", false);;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSelfJoinActivity(_ctx, this, _probeManager, forceSpill);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSelfJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSelfJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerLookupJoinActivity : public CRoxieServerTwoInputActivity
  {
  private:
      class LookupTable : public CInterface
      {
      public:
          LookupTable(IHThorHashJoinArg &helper)
          : leftRightCompare(helper.queryCompareLeftRight()), rightCompare(helper.queryCompareRight()),
            leftHash(helper.queryHashLeft()), rightHash(helper.queryHashRight())
          {
              size = 0;
          }
          virtual const void *find(const void * left) const = 0;
          virtual const void *findNext(const void * left) const = 0;
  
      protected:
          ICompare * leftRightCompare;
          ICompare * rightCompare;
          IHash * leftHash;
          IHash * rightHash;
          unsigned size;
      };
  
      class DedupLookupTable : public LookupTable
      {
      public:
          DedupLookupTable(ConstPointerArray &rightRows, IHThorHashJoinArg &helper)
          : LookupTable(helper)
          {
              size = (4*rightRows.length())/3 + 1;
              table = (const void * *)calloc(size, sizeof(void *)); // This should probably be allocated from roxiemem (and size rounded up to actual available size)
              ForEachItemIn(idx, rightRows)
                  add(rightRows.item(idx));
          }
  
          ~DedupLookupTable()
          {
              roxiemem::ReleaseRoxieRowArray(size, table);
              free(table);
          }
  
          virtual const void *find(const void * left) const
          {
              unsigned index = leftHash->hash(left) % size;
              unsigned start = index;
              while (table[index])
              {
                  if(leftRightCompare->docompare(left, table[index]) == 0)
                      return table[index];
                  index++;
                  if (index==size)
                      index = 0;
                  if (index==start)
                      throw MakeStringException(ROXIE_JOIN_ERROR, "Internal error in lookup join activity (hash table full on lookup)");
              }
              return NULL;
          }
  
          virtual const void *findNext(const void * left) const
          {
              return NULL;
          }
  
      protected:
          void add(const void * right)
          {
              unsigned index = rightHash->hash(right) % size;
              unsigned start = index;
              while (table[index])
              {
                  if (rightCompare->docompare(table[index], right) == 0)
                  {
                      ReleaseRoxieRow(right);
                      return;
                  }
                  index++;
                  if (index==size)
                      index = 0;
                  if (index==start)
                      throw MakeStringException(ROXIE_JOIN_ERROR, "Internal error in lookup join activity (hash table full on add)");
              }
              table[index] = right;
          }
  
          const void * * table;
      };
  
      class FewLookupTable : public LookupTable
      {
      public:
          FewLookupTable(ConstPointerArray &rightRows, IHThorHashJoinArg &helper)
          : LookupTable(helper)
          {
              size = (4*rightRows.length())/3 + 1;
              table = (const void * *)calloc(size, sizeof(void *)); // This should probably be allocated from roxiemem
              findex = fstart = BadIndex;
              ForEachItemIn(idx, rightRows)
                  add(rightRows.item(idx));
          }
  
          ~FewLookupTable()
          {
              roxiemem::ReleaseRoxieRowArray(size, table);
              free(table);
          }
  
          virtual const void *find(const void * left) const
          {
              fstart = leftHash->hash(left) % size;
              findex = fstart;
              return doFind(left);
          }
          virtual const void *findNext(const void * left) const
          {
              if (findex == BadIndex)
                  return NULL;
              advance();
              return doFind(left);
          }
      protected:
          void add(const void * right)
          {
              unsigned start = rightHash->hash(right) % size;
              unsigned index = start;
              while (table[index])
              {
                  index++;
                  if (index==size)
                      index = 0;
                  if (index==start)
                      throwUnexpected(); //table is full, should never happen
              }
              table[index] = right;
          }
          void advance() const
          {
              findex++;
              if(findex==size)
                  findex = 0;
              if(findex==fstart)
                  throw MakeStringException(ROXIE_JOIN_ERROR, "Internal error in lookup join activity (hash table full on lookup)");
          }
          const void *doFind(const void * left) const
          {
              while(table[findex])
              {
                  if (leftRightCompare->docompare(left, table[findex]) == 0)
                      return table[findex];
                  advance();
              }
              findex = BadIndex;
              return NULL;
          }
  
          const void * * table;
          unsigned mutable fstart;
          unsigned mutable findex;
          static unsigned const BadIndex;
      };
  
      class ManyLookupTable : public LookupTable
      {
      public:
          ManyLookupTable(ConstPointerArray &rightRows, IHThorHashJoinArg &helper)
          : LookupTable(helper)
          {
              rightRows.swapWith(rowtable);
              UInt64Array groups;
              unsigned numRows = rowtable.length();
              if (numRows)
              {
                  unsigned groupStart = 0;
                  const void *groupStartRow = rowtable.item(0);
                  for (unsigned i=1; i < numRows; i++)
                  {
                      const void *thisRow = rowtable.item(i);
                      if (rightCompare->docompare(groupStartRow, thisRow))
                      {
                          groups.append(makeint64(groupStart, i-groupStart));
                          groupStart = i;
                          groupStartRow = thisRow;
                      }
                  }
                  groups.append(makeint64(groupStart, numRows-groupStart));
              }
              size = (4*groups.length())/3 + 1;
              table = (__uint64 *) calloc(size, sizeof(__uint64)); // This should probably be allocated from roxiemem
              ForEachItemIn(idx, groups)
              {
                  unsigned __int64 group = groups.item(idx);
                  unsigned groupstart = high(group);
                  const void *row = rowtable.item(groupstart);
                  add(row, group);
              }
          }
  
          ~ManyLookupTable()
          {
              roxiemem::ReleaseRoxieRows(rowtable);
              free(table);
          }
  
          void add(const void *row, unsigned __int64 group)
          {
              unsigned start = rightHash->hash(row) % size;
              unsigned index = start;
              while (table[index])
              {
                  index++;
                  if (index==size)
                      index = 0;
                  if (index==start)
                      throwUnexpected(); //table is full, should never happen
              }
              table[index] = group;
          }
  
          virtual const void *find(const void * left) const
          {
              unsigned index = leftHash->hash(left) % size;
              unsigned start = index;
              while (table[index])
              {
                  __uint64 group = table[index];
                  currentMatch = high(group);
                  const void *right = rowtable.item(currentMatch);
                  if (leftRightCompare->docompare(left, right) == 0)
                  {
                      currentMatch++;
                      matchCount = low(group) - 1;
                      return right;
                  }
                  index++;
                  if (index==size)
                      index = 0;
                  if (index==start)
                      throw MakeStringException(ROXIE_JOIN_ERROR, "Internal error in lookup join activity (hash table full on lookup)");
              }
              matchCount = 0;
              return NULL;
          }
  
          virtual const void *findNext(const void * left) const
          {
              if (!matchCount)
                  return NULL;
              matchCount--;
              return rowtable.item(currentMatch++);
          }
  
      protected:
          __uint64 *table;
          ConstPointerArray rowtable;
          mutable unsigned currentMatch = 0;
          mutable unsigned matchCount = 0;
      };
  
      IHThorHashJoinArg &helper;
      bool leftOuterJoin;
      bool exclude;
      bool eog;
      bool many;
      bool dedupRHS;
      bool useFewTable;
      bool matchedGroup;
      const void *left;
      OwnedConstRoxieRow defaultRight;
      Owned<LookupTable> table;
      unsigned keepLimit;
      unsigned atmostLimit;
      unsigned atmostsTriggered;
      unsigned limitLimit;
      unsigned joinCounter;
      bool limitFail;
      bool limitOnFail;
      bool hasGroupLimit;
      bool isSmartJoin;
      unsigned keepCount;
      bool gotMatch;
      bool cloneLeft;
      ConstPointerArray rightGroup;
      aindex_t rightGroupIndex;
      Owned<IException> failingLimit;
      ConstPointerArray filteredRight;
      ThorActivityKind activityKind;
      Owned<IEngineRowAllocator> defaultRightAllocator;
      
      void createDefaultRight()
      {
          if (!defaultRight)
          {
              if (!defaultRightAllocator)
                  defaultRightAllocator.setown(createRowAllocator(input1->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
              size32_t thisSize = helper.createDefaultRight(rowBuilder);
              defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
  public:
      CRoxieServerLookupJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _useFewTable)
          : CRoxieServerTwoInputActivity(_ctx, _factory, _probeManager), helper((IHThorHashJoinArg &)basehelper), useFewTable(_useFewTable)
      {
          unsigned joinFlags = helper.getJoinFlags();
          leftOuterJoin = (joinFlags & JFleftouter) != 0;
          assertex((joinFlags & JFrightouter) == 0);
          exclude = (joinFlags & JFexclude) != 0;
          many = (joinFlags & JFmanylookup) != 0;
          cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
          dedupRHS = (joinFlags & (JFmanylookup | JFmatchrequired | JFtransformMaySkip)) == 0; // optimisation: can implicitly dedup RHS unless is many lookup, or match required, or transform may skip
          left = NULL;
          activityKind = factory->getKind();
          eog = false;
          matchedGroup = false;
          gotMatch = false;
          keepLimit = 0;
          keepCount = 0;
          joinCounter = 0;
          atmostLimit = 0;
          atmostsTriggered = 0;
          limitLimit = 0;
          rightGroupIndex = 0;
          hasGroupLimit = false;
          isSmartJoin = (joinFlags & JFsmart) != 0;
          getLimitType(joinFlags, limitFail, limitOnFail);
      }
  
      void loadRight()
      {
          ConstPointerArray rightset;
          try
          {
              const void * next;
              while(true)
              {
                  next = inputStream1->ungroupedNextRow();
                  if(!next)
                      break;
                  rightset.append(next);
              }
              if (!dedupRHS)
              {
                  if (useFewTable)
                  {
                      table.setown(new FewLookupTable(rightset, helper));  // NOTE - takes ownership of rightset
                  }
                  else
                  {
                      if (!helper.isRightAlreadySorted())
                      {
                          if (helper.getJoinFlags() & JFunstable)
                          {
                              qsortvec(const_cast<void * *>(rightset.getArray()), rightset.ordinality(), *helper.queryCompareRight());
                          }
                          else
                          {
                              unsigned rightord = rightset.ordinality();
                              MemoryAttr tempAttr(rightord*sizeof(void **)); // Temp storage for stable sort. This should probably be allocated from roxiemem
                              void **temp = (void **) tempAttr.bufferBase();
                              void **_rows = const_cast<void * *>(rightset.getArray());
                              msortvecstableinplace(_rows, rightord, *helper.queryCompareRight(), temp);
                          }
                      }
                      table.setown(new ManyLookupTable(rightset, helper));  // NOTE - takes ownership of rightset
                  }
              }
              else
              {
                  table.setown(new DedupLookupTable(rightset, helper)); // NOTE - takes ownership of rightset
              }
          }
          catch (...)
          {
              roxiemem::ReleaseRoxieRows(rightset);
              throw;
          }
      };
  
      virtual void reset()
      {
          if (atmostsTriggered)
              noteStatistic(StNumAtmostTriggered, atmostsTriggered);
          CRoxieServerTwoInputActivity::reset();
          ReleaseClearRoxieRow(left);
          defaultRight.clear();
          table.clear();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eog = false;
          matchedGroup = false;
          left = NULL;
          CRoxieServerTwoInputActivity::start(parentExtractSize, parentExtract, paused);
          keepLimit = helper.getKeepLimit();
          if(keepLimit==0) keepLimit = static_cast<unsigned>(-1);
          atmostsTriggered = 0;
          atmostLimit = helper.getJoinLimit();
          limitLimit = helper.getMatchAbortLimit();
          hasGroupLimit = ((atmostLimit > 0) || (limitLimit > 0));
          if(atmostLimit==0) atmostLimit = static_cast<unsigned>(-1);
          if(limitLimit==0) limitLimit = static_cast<unsigned>(-1);
          getLimitType(helper.getJoinFlags(), limitFail, limitOnFail);
          switch (activityKind)
          {
          case TAKlookupjoin:
          case TAKlookupdenormalizegroup:
          case TAKsmartjoin:
          case TAKsmartdenormalizegroup:
              if (leftOuterJoin)
                  createDefaultRight();
              break;
          }
          if (limitOnFail)
              createDefaultRight();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (!idx && (helper.getJoinFlags() & JFparallel) != 0)
          {
              puller.setown(new CRoxieServerReadAheadInput(ctx, ctx->queryOptions().parallelJoinPreload));
              puller->setInput(0, _sourceIdx, _in);
              _in = puller;
              _sourceIdx = 0;
          }
          CRoxieServerTwoInputActivity::setInput(idx, _sourceIdx, _in);
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if(!table)
              loadRight();
          switch (activityKind)
          {
              case TAKlookupjoin:
              case TAKsmartjoin:
                  return nextRowJoin();
              case TAKlookupdenormalize:
              case TAKlookupdenormalizegroup:
              case TAKsmartdenormalize:
              case TAKsmartdenormalizegroup:
                  return nextRowDenormalize();
          }
          throwUnexpected();
      }
  
  private:
      const void * nextRowJoin()
      {
          if(!table)
              loadRight();
          while(true)
          {
              const void * right = NULL;
              if(!left)
              {
                  left = inputStream->nextRow();
                  keepCount = keepLimit;
                  joinCounter = 0;
                  if(!left)
                  {
                      if (isSmartJoin)
                          left = inputStream->nextRow();
  
                      if (!left)
                      {
                          if(matchedGroup || eog)
                          {
                              matchedGroup = false;
                              eog = true;
                              return NULL;
                          }
                          eog = true;
                          continue;
                      }
                  }
                  eog = false;
                  gotMatch = false;
                  right = getRightFirst();
              }
              else
                  right = getRightNext();
              const void * ret = NULL;
              if(failingLimit)
              {
                  ret = joinException(left, failingLimit);
              }
              else
              {
                  while(right)
                  {
                      if(helper.match(left, right))
                      {
                          gotMatch = true;
                          if(exclude)
                              break;
                          ret = joinRecords(left, right, ++joinCounter, JTFmatchedleft|JTFmatchedright);
                          if(ret)
                          {
                              processed++;
                              break;
                          }
                      }
                      right = getRightNext();
                      ret = NULL;
                  }
                  if(leftOuterJoin && !gotMatch)
                  {
                      ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
                      gotMatch = true;
                  }
              }
              if(ret)
              {
                  matchedGroup = true;
                  processed++;
                  if(!many || (--keepCount == 0) || failingLimit)
                  {
                      ReleaseClearRoxieRow(left);
                      failingLimit.clear();
                  }
                  return ret;
              }
              ReleaseClearRoxieRow(left);
          }
      }
  
      const void * nextRowDenormalize()
      {
          while(true)
          {
              left = inputStream->nextRow();
              if(!left)
              {
                  if (!matchedGroup || isSmartJoin)
                      left = inputStream->nextRow();
  
                  if (!left)
                  {
                      matchedGroup = false;
                      return NULL;
                  }
              }
              gotMatch = false;
  
              const void * right = getRightFirst();
              const void * ret = NULL;
              if (failingLimit)
                  ret = joinException(left, failingLimit);
              else if (activityKind == TAKlookupdenormalize || activityKind == TAKsmartdenormalize)
              {
                  OwnedConstRoxieRow newLeft;
                  newLeft.set(left);
                  unsigned rowSize = 0;
                  unsigned leftCount = 0;
                  keepCount = keepLimit;
                  while (right)
                  {
                      try
                      {
                          if (helper.match(left, right))
                          {
                              gotMatch = true;
                              if (exclude)
                                  break;
  
                              RtlDynamicRowBuilder rowBuilder(rowAllocator);
                              unsigned thisSize = helper.transform(rowBuilder, newLeft, right, ++leftCount, JTFmatchedleft|JTFmatchedright);
                              if (thisSize)
                              {
                                  rowSize = thisSize;
                                  newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
                              }
                              if(!many || (--keepCount == 0))
                                  break;
                          }
                          right = getRightNext();
                      }
                      catch (IException *E)
                      {
                          throw makeWrappedException(E);
                      }
                  }
                  if (rowSize)
                      ret = newLeft.getClear();
                  else if (leftOuterJoin && !gotMatch)
                  {
                      ret = left;
                      left = NULL;
                  }
              }
              else
              {
                  filteredRight.kill();
                  keepCount = keepLimit;
                  while (right)
                  {
                      if (helper.match(left, right))
                      {
                          gotMatch = true;
                          if(exclude)
                              break;
                          filteredRight.append(right);
                          if(!many || (--keepCount == 0))
                              break;
                      }
                      right = getRightNext();
                  }
  
                  if((filteredRight.ordinality() > 0) || (leftOuterJoin && !gotMatch))
                      ret = denormalizeRecords(left, filteredRight, JTFmatchedleft);
                  filteredRight.kill();
              }
              ReleaseRoxieRow(left);
              left = NULL;
              failingLimit.clear();
              if(ret)
              {
                  matchedGroup = true;
                  processed++;
                  return ret;
              }
          }
      }
  
      const void * joinRecords(const void * left, const void * right, unsigned counter, unsigned flags)
      {
          if (cloneLeft)
          {
              LinkRoxieRow(left);
              return left;
          }
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned outSize = helper.transform(rowBuilder, left, right, counter, flags);
              if (outSize)
                  return rowBuilder.finalizeRowClear(outSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * joinException(const void * left, IException * except)
      {
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned outSize = helper.onFailTransform(rowBuilder, left, defaultRight, except, JTFmatchedleft);
              if (outSize)
                  return rowBuilder.finalizeRowClear(outSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * denormalizeRecords(const void * left, ConstPointerArray & rows, unsigned flags)
      {
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned numRows = rows.ordinality();
              const void * right = numRows ? rows.item(0) : defaultRight.get();
              if (numRows>0)
                  flags |= JTFmatchedright;
              unsigned outSize = helper.transform(rowBuilder, left, right, numRows, (const void * *)rows.getArray(), flags);
              if (outSize)
                  return rowBuilder.finalizeRowClear(outSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * getRightFirst() { if(hasGroupLimit) return fillRightGroup(); else return table->find(left); }
      const void * getRightNext() { if(hasGroupLimit) return readRightGroup(); else return table->findNext(left); }
      const void * readRightGroup() { if(rightGroup.isItem(rightGroupIndex)) return rightGroup.item(rightGroupIndex++); else return NULL; }
  
      const void *fillRightGroup()
      {
          rightGroup.kill();
          for(const void * right = table->find(left); right; right = table->findNext(left))
          {
              rightGroup.append(right);
              if(rightGroup.ordinality() > limitLimit)
              {
                  if(limitFail)
                      failLimit();
                  gotMatch = true;
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  if(limitOnFail)
                  {
                      assertex(!failingLimit);
                      try
                      {
                          failLimit();
                      }
                      catch(IException * e)
                      {
                          failingLimit.setown(e);
                      }
                      assertex(failingLimit != NULL);
                  }
                  else
                  {
                      rightGroup.kill();
                  }
                  break;
              }
              if(rightGroup.ordinality() > atmostLimit)
              {
                  atmostsTriggered++;
                  rightGroup.kill();
                  break;
              }
          }
          rightGroupIndex = 0;
          return readRightGroup();
      }
  
      void failLimit()
      {
          helper.onMatchAbortLimitExceeded();
          CommonXmlWriter xmlwrite(XWFtrim|XWFopt );
          if(!ctx->isBlind() && input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
          {
              input->queryOutputMeta()->toXML(static_cast<const unsigned char *>(left), xmlwrite);
          }
          throw MakeStringException(ROXIE_TOO_MANY_RESULTS, "More than %u match candidates in join %d for row %s", limitLimit, queryId(), xmlwrite.str());
      }
      
  };
  
  unsigned const CRoxieServerLookupJoinActivity::FewLookupTable::BadIndex(static_cast<unsigned>(-1));
  
  class CRoxieServerLookupJoinActivityFactory : public CRoxieServerJoinActivityFactory
  {
  public:
      CRoxieServerLookupJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode)
      {
          Owned<IHThorHashJoinArg> helper = (IHThorHashJoinArg *) helperFactory();
          useFewTable = _graphNode.getPropBool("hint[@name='usefewtable']/@value", false);
          if((helper->getJoinFlags() & (JFfirst | JFfirstleft | JFfirstright | JFslidingmatch)) != 0)
              throw MakeStringException(ROXIE_INVALID_FLAGS, "Invalid flags for lookup join activity"); // code generator should never create such an activity
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLookupJoinActivity(_ctx, this, _probeManager, useFewTable);
      }
  protected:
      bool useFewTable;
  };
  
  IRoxieServerActivityFactory *createRoxieServerLookupJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerLookupJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerAllJoinActivity : public CRoxieServerTwoInputActivity
  {
  private:
      IHThorAllJoinArg &helper;
      bool leftOuterJoin;
      bool rightOuterJoin;
      bool exclude;
      OwnedConstRoxieRow defaultRight;
      OwnedConstRoxieRow defaultLeft;
      Owned<IEngineRowAllocator> defaultRightAllocator;
      Owned<IEngineRowAllocator> defaultLeftAllocator;
      const void *left;
      unsigned countForLeft;
      ConstPointerArray rightset;
      BoolArray matchedRight; // MORE - could use a bitset...
      unsigned keepLimit;
      bool started;
      bool eog;
      bool eos;
      bool matchedLeft;
      bool matchedGroup;
      bool leftIsGrouped;
      bool cloneLeft;
      unsigned rightIndex;
      unsigned joinCounter;
      unsigned rightOrdinality;
      ThorActivityKind activityKind;
      ConstPointerArray filteredRight;
  
      void createDefaultLeft()
      {
          if (!defaultLeft)
          {
              if (!defaultLeftAllocator)
                  defaultLeftAllocator.setown(createRowAllocator(input->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultLeftAllocator);
              size32_t thisSize = helper.createDefaultLeft(rowBuilder);
              defaultLeft.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
      void createDefaultRight()
      {
          if (!defaultRight)
          {
              if (!defaultRightAllocator)
                  defaultRightAllocator.setown(createRowAllocator(input1->queryOutputMeta()));
  
              RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
              size32_t thisSize = helper.createDefaultRight(rowBuilder);
              defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
  public:
      CRoxieServerAllJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerTwoInputActivity(_ctx, _factory, _probeManager), helper((IHThorAllJoinArg &)basehelper)
      {
          unsigned joinFlags = helper.getJoinFlags();
          leftOuterJoin = (joinFlags & JFleftouter) != 0;
          rightOuterJoin = (joinFlags & JFrightouter) != 0;
          cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
          keepLimit = (unsigned) -1;
          exclude = (joinFlags & JFexclude) != 0;
          left = NULL;
          started = true;
          eog = false;
          eos = false;
          matchedLeft = false;
          matchedGroup = false;
          activityKind = factory->getKind();
          rightIndex = 0;
          rightOrdinality = 0;
          leftIsGrouped = false;
          countForLeft = 0;
          joinCounter = 0;
      }
  
      virtual void reset()
      {
          defaultRight.clear();
          defaultLeft.clear();
          ReleaseRoxieRows(rightset);
          if (left)
          {
              ReleaseRoxieRow(left);
              left = nullptr;
          }
          matchedRight.kill();
          CRoxieServerTwoInputActivity::reset();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eog = false;
          eos = false;
          matchedLeft = false;
          matchedGroup = false;
          started = false;
          left = NULL;
  
          CRoxieServerTwoInputActivity::start(parentExtractSize, parentExtract, paused);
          keepLimit = helper.getKeepLimit();
          if(keepLimit==0)
              keepLimit = (unsigned) -1;
          countForLeft = keepLimit;
          joinCounter = 0;
          leftIsGrouped = input->queryOutputMeta()->isGrouped();
          if((activityKind==TAKalljoin || activityKind==TAKalldenormalizegroup) && leftOuterJoin)
              createDefaultRight();
          if(rightOuterJoin)
              createDefaultLeft();
      }
  
      void loadRight()
      {
          const void * next;
          while(true)
          {
              next = inputStream1->ungroupedNextRow();
              if(!next)
                  break;
              rightset.append(next);
              matchedRight.append(false);
          }
          rightIndex = 0;
          joinCounter = 0;
          rightOrdinality = rightset.ordinality();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (!idx && (helper.getJoinFlags() & JFparallel) != 0)
          {
              puller.setown(new CRoxieServerReadAheadInput(ctx, ctx->queryOptions().parallelJoinPreload));
              puller->setInput(0, _sourceIdx, _in);
              _in = puller;
              _sourceIdx = 0;
          }
          CRoxieServerTwoInputActivity::setInput(idx, _sourceIdx, _in);
      }
  
      const void * joinRecords(const void * left, const void * right, unsigned counter, unsigned flags)
      {
          // MORE - could share some code with lookup join
          if (cloneLeft)
          {
              LinkRoxieRow(left);
              return left;
          }
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned outSize = helper.transform(rowBuilder, left, right, counter, flags);
              if (outSize)
                  return rowBuilder.finalizeRowClear(outSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      const void * denormalizeRecords(const void * curLeft, ConstPointerArray & rows, unsigned flags)
      {
          try
          {
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              unsigned numRows = rows.ordinality();
              const void * right = numRows ? rows.item(0) : defaultRight.get();
              if (numRows>0)
                  flags |= JTFmatchedright;
              unsigned outSize = helper.transform(rowBuilder, curLeft, right, numRows, rows.getArray(), flags);
              if (outSize)
                  return rowBuilder.finalizeRowClear(outSize);
              else
                  return NULL;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if(!started)
          {
              started = true;
              left = inputStream->nextRow();
              matchedLeft = false;
              countForLeft = keepLimit;
              joinCounter = 0;
              if(left == NULL)
              {
                  eos = true;
                  return NULL;
              }
              loadRight();
          }
  
          const void * ret;   
          const void * right;
          if(eos)
              return NULL;
  
          while(true)
          {
              ret = NULL;
  
              if((rightIndex == rightOrdinality) || (countForLeft==0))
              {
                  if(leftOuterJoin && left && !matchedLeft)
                  {
                      switch(activityKind)
                      {
                      case TAKalljoin:
                          ret = joinRecords(left, defaultRight, 0, JTFmatchedleft);
                          break;
                      case TAKalldenormalize:
                          ret = left;
                          left = NULL;
                          break;
                      case TAKalldenormalizegroup:
                          filteredRight.kill();
                          ret = denormalizeRecords(left, filteredRight, JTFmatchedleft);
                          break;
                      default:
                          throwUnexpected();
                      }
                  }
                  rightIndex = 0;
                  joinCounter = 0;
                  ReleaseRoxieRow(left);
                  left = NULL;
                  if(ret)
                  {
                      matchedGroup = true;
                      processed++;
                      return ret;
                  }
              }
  
              if(!left)
              {
                  left = inputStream->nextRow();
                  matchedLeft = false;
                  countForLeft = keepLimit;
                  joinCounter = 0;
              }
              if(!left)
              {
                  if(eog)
                  {
                      eos = true;
                      matchedGroup = false;
                      return NULL;
                  }
                  eog = true;
                  if (matchedGroup && leftIsGrouped)
                  {
                      matchedGroup = false;
                      return NULL;
                  }
                  matchedGroup = false;
                  continue;
              }
  
              eog = false;
              switch(activityKind)
              {
              case TAKalljoin:
                  while(rightIndex < rightOrdinality)
                  {
                      right = rightset.item(rightIndex);
                      if(helper.match(left, right))
                      {
                          matchedLeft = true;
                          matchedRight.replace(true, rightIndex);
                          if(!exclude)
                              ret = joinRecords(left, right, ++joinCounter, JTFmatchedleft|JTFmatchedright);
                      }
                      rightIndex++;
                      if(ret)
                      {
                          countForLeft--;
                          matchedGroup = true;
                          processed++;
                          return ret;
                      }
                  }
                  break;
              case TAKalldenormalize:
                  {
                      OwnedConstRoxieRow newLeft;
                      newLeft.set(left);
                      unsigned rowSize = 0;
                      unsigned leftCount = 0;
                      while((rightIndex < rightOrdinality) && countForLeft)
                      {
                          right = rightset.item(rightIndex);
                          if(helper.match(left, right))
                          {
                              matchedLeft = true;
                              matchedRight.replace(true, rightIndex);
                              if(!exclude)
                              {
                                  try
                                  {
                                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                                      unsigned thisSize = helper.transform(rowBuilder, newLeft, right, ++leftCount, JTFmatchedleft|JTFmatchedright);
                                      if(thisSize)
                                      {
                                          rowSize = thisSize;
                                          newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
                                          --countForLeft;
                                      }
                                  }
                                  catch (IException *e)
                                  {
                                      throw makeWrappedException(e);
                                  }
                              }
                          }
                          rightIndex++;
                      }
                      if(rowSize)
                      {
                          processed++;
                          return newLeft.getClear();
                      }
                  }
                  break;
              case TAKalldenormalizegroup:
                  filteredRight.kill();
                  while((rightIndex < rightOrdinality) && countForLeft)
                  {
                      right = rightset.item(rightIndex);
                      if(helper.match(left, right))
                      {
                          matchedLeft = true;
                          matchedRight.replace(true, rightIndex);
                          filteredRight.append(right);
                          --countForLeft;
                      }
                      ++rightIndex;
                  }
                  if(!exclude && filteredRight.ordinality())
                  {
                      ret = denormalizeRecords(left, filteredRight, JTFmatchedleft);
                      filteredRight.kill();
                      if(ret)
                      {
                          processed++;
                          return ret;
                      }
                  }
                  break;
              default:
                  throwUnexpected();
              }
          }
      }
  
  };
  
  class CRoxieServerAllJoinActivityFactory : public CRoxieServerJoinActivityFactory
  {
  public:
      CRoxieServerAllJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode)
      {
          Owned<IHThorAllJoinArg> helper = (IHThorAllJoinArg *) helperFactory();
          if((helper->getJoinFlags() & (JFfirst | JFfirstleft | JFfirstright)) != 0)
              throw MakeStringException(ROXIE_INVALID_FLAGS, "Invalid flags for join all activity"); // code generator should never create such an activity
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerAllJoinActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerAllJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerAllJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerTopNActivity : public CRoxieServerLateStartActivity
  {
      unsigned limit;
      bool hasBest;
      bool eoi;
      const void **sorted;
      unsigned sortedCount;
      unsigned curIndex;
      IHThorTopNArg &helper;
      ICompare &compare;
  
  public:
      CRoxieServerTopNActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerLateStartActivity(_ctx, _factory, _probeManager), helper((IHThorTopNArg &)basehelper), compare(*helper.queryCompare())
      {
          sorted = NULL;
          sortedCount = 0;
          curIndex = 0;
          limit = 0;
          eoi = false;
          hasBest = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          assertex(sorted == NULL);
          sortedCount = 0;
          curIndex = 0;
          eoi = false;
          
          CRoxieServerLateStartActivity::start(parentExtractSize, parentExtract, paused);
          limit = (unsigned) helper.getLimit();
          hasBest = helper.hasBest();
          lateStart(parentExtractSize, parentExtract, limit > 0);
          // MORE - should we use an expanding array instead?
          if (limit > 0)
              sorted = (const void **) ctx->queryRowManager().allocate((limit+1) * sizeof(const void *), activityId); 
      }
  
      virtual void reset()
      {
          if (sorted)
          {
              roxiemem::ReleaseRoxieRowRange(sorted, curIndex, sortedCount);
              curIndex = 0;
              sortedCount = 0;
              ReleaseRoxieRow(sorted);
              sorted = NULL;
          }
          CRoxieServerLateStartActivity::reset();
      }
  
      bool abortEarly()
      {
          if (hasBest && (sortedCount == limit))
          {
              int compare = helper.compareBest(sorted[sortedCount-1]);
              if (compare == 0)
              {
                  if (meta.isGrouped())
                  {
                      //MORE: This would be more efficient if we had a away of skipping to the end of the incoming group.
                      const void * next;
                      while ((next = inputStream->nextRow()) != NULL)
                          ReleaseRoxieRow(next);
                  }
                  else
                      eoi = true;
                  return true;
              }
  
              //This only checks the lowest element - we could check all elements inserted, but it would increase the number of compares
              if (compare < 0)
                  throw MakeStringException(ROXIE_TOPN_ROW_ERROR, "TOPN: row found that exceeds the best value");
          }
          return false;
      }
  
      void getSorted()
      {
          curIndex = 0;
          sortedCount = 0;
          if(eoi)
              return;
          const void * next;
          while ((next = inputStream->nextRow()) != NULL)
          {
              if (sortedCount < limit)
              {
                  binary_vec_insert_stable(next, sorted, sortedCount, compare);
                  sortedCount++;
                  if(abortEarly())
                      return;
              }
              else
              {
                  if(limit && compare.docompare(sorted[sortedCount-1], next) > 0) // MORE - if stability is an issue, need to consider whether this should be > or >=
                  {
                      binary_vec_insert_stable(next, sorted, sortedCount, compare); // MORE - not sure this is stable!
                      ReleaseRoxieRow(sorted[sortedCount]);
                      if(abortEarly())
                          return;
                  }
                  else
                  {
                      ReleaseRoxieRow(next); // do not bother with insertion sort if we know next will fall off the end
                  }
              }
          }
      }
  
      virtual const void * nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          if (curIndex >= sortedCount)
          {
              bool eog = sortedCount != 0;
              getSorted();
              if(sortedCount == 0)
              {
                  eof = true;
                  return NULL;
              }
              if (eog)
                  return NULL;
          }
          processed++;
          return sorted[curIndex++];
      }
  
  };
  
  class CRoxieServerTopNActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerTopNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerTopNActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerTopNActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerTopNActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerLimitActivity : public CRoxieServerActivity
  {
  protected:
      unsigned __int64 rowLimit;
      IHThorLimitArg &helper;
  
  public:
      CRoxieServerLimitActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorLimitArg &)basehelper)
      {
          rowLimit = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          rowLimit = helper.getRowLimit();  // could conceivably depend on context so should not compute any earlier than this
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * ret = inputStream->nextRow();
          if (ret)
          {
              processed++;
              if (processed > rowLimit)
              {
                  ReleaseRoxieRow(ret);
                  if (traceLevel > 4)
                      DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                  helper.onLimitExceeded();
              }
          }
          return ret;
      }
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          const void * ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (ret)
          {
              if (wasCompleteMatch)
                  processed++;
              if (processed > rowLimit)
              {
                  ReleaseRoxieRow(ret);
                  if (traceLevel > 4)
                      DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                  helper.onLimitExceeded();
              }
          }
          return ret;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          //Do not reset the rowLimit
          inputStream->resetEOF();
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  };
  
  class CRoxieServerLimitActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerLimitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerLimitActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerLimitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerLimitActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerSkipLimitActivity : public CRoxieServerLimitActivity
  {
      ConstPointerArray buff;
      bool started = false;
      unsigned index = 0;
      bool isFail = false;
  
  public:
      CRoxieServerSkipLimitActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _onFail)
          : CRoxieServerLimitActivity(_ctx, _factory, _probeManager)
      {
          started = false;
          index = 0;
          isFail = _onFail;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          started = false;
          index = 0;
          CRoxieServerLimitActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          roxiemem::ReleaseRoxieRowRange(buff.getArray(), index, buff.ordinality());
          buff.kill();
          index = 0;
          started = false;
          CRoxieServerLimitActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!started)
              pullInput();
          if (buff.isItem(index))
          {
              const void * next = buff.item(index++);
              if(next)
                  processed++;
              return next;
          }
          return NULL;
      }
  
  protected:
      void pullInput()
      {
          unsigned count = 0;
          for (;;)
          {
              const void * next = inputStream->nextRow();
              if (next == NULL)
              {
                  next = inputStream->nextRow();
                  if(next == NULL)
                      break;
                  buff.append(NULL);
              }
              count++;
              if (count > rowLimit)
              {
                  ReleaseRoxieRow(next);
                  ReleaseRoxieRows(buff);
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  if (isFail)
                  {
                      ensureRowAllocator();
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      size32_t outSize = helper.transformOnLimitExceeded(rowBuilder);
                      if (outSize)
                          buff.append(rowBuilder.finalizeRowClear(outSize));
                  }
                  break;
              }
              buff.append(next);
          }
          started = true;
      }
  };
  
  class CRoxieServerSkipLimitActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSkipLimitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSkipLimitActivity(_ctx, this, _probeManager, kind==TAKcreaterowlimit);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSkipLimitActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSkipLimitActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerCatchActivity : public CRoxieServerActivity
  {
      IHThorCatchArg &helper;
  
  public:
      CRoxieServerCatchActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorCatchArg &)basehelper)
      {
      }
  
      // MORE - you could argue we should catch exceptions in START too, though not clear what to do with them
      // See CRoxieServerSkipCatchActivity::start
      // This is less significant now that we properly distinguish start exceptions from read exceptions in ThroughSpill activity
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          try
          {
              const void *ret = inputStream->nextRow();
              if (ret)
                  processed++;
              return ret;
          }
          catch (IException *E)
          {
              E->Release();
              helper.onExceptionCaught();
          }
          catch (...)
          {
              helper.onExceptionCaught();
          }
          throwUnexpected(); // onExceptionCaught should have thrown something
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          try
          {
              ActivityTimer t(activityStats, timeActivities);
              const void * ret = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
              if (ret && wasCompleteMatch)
                  processed++;
              return ret;
          }
          catch (IException *E)
          {
              E->Release();
              helper.onExceptionCaught();
          }
          catch (...)
          {
              helper.onExceptionCaught();
          }
          throwUnexpected(); // onExceptionCaught should have thrown something
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) 
      { 
          return input->gatherConjunctions(collector); 
      }
  
      virtual void resetEOF() 
      { 
          inputStream->resetEOF(); // MORE - why not in base class?
      }
  
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  };
  
  class CRoxieServerSkipCatchActivity : public CRoxieServerActivity
  {
      ConstPointerArray buff;
      bool started;
      unsigned index;
      IHThorCatchArg &helper;
      bool createRow;
  
  public:
      CRoxieServerSkipCatchActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _createRow)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorCatchArg &)basehelper), createRow(_createRow)
      {
          started = false;
          index = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          started = false;
          index = 0;
          try
          {
              CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          }
          catch (IException *E)
          {
              ensureCreated();
              onException(E);
              started = true;
          }
          catch (...)
          {
              ensureCreated();
              onException(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught"));
              started = true;
          }
      }
  
      virtual void reset()
      {
          roxiemem::ReleaseRoxieRowRange(buff.getArray(), index, buff.ordinality());
          buff.kill();
          index = 0;
          started = false;
          CRoxieServerActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!started)
              pullInput();
          if (buff.isItem(index))
          {
              const void * next = buff.item(index++);
              if(next)
                  processed++;
              return next;
          }
          return NULL;
      }
  
  protected:
      void onException(IException *E)
      {
          inputStream->stop();
          ReleaseRoxieRows(buff);
          if (createRow)
          {
              ensureRowAllocator();
              RtlDynamicRowBuilder rowBuilder(rowAllocator);
              size32_t outSize = helper.transformOnExceptionCaught(rowBuilder, E);
              if (outSize)
                  buff.append(rowBuilder.finalizeRowClear(outSize));
          }
          E->Release();
      }
  
      void pullInput()
      {
          try
          {
              bool EOGseen = false;
              for (;;)
              {
                  const void * next = inputStream->nextRow();
                  buff.append(next);
                  if (next == NULL)
                  {
                      if (EOGseen)
                          break;
                      EOGseen = true;
                  }
                  else
                      EOGseen = false;
              }
          }
          catch (IException *E)
          {
              onException(E);
          }
          catch (...)
          {
              onException(MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception caught"));
          }
          started = true;
      }
  };
  
  class CRoxieServerCatchActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerCatchActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          switch (kind)
          {
          case TAKcatch:
              return new CRoxieServerCatchActivity(_ctx, this, _probeManager);
          case TAKskipcatch:
              return new CRoxieServerSkipCatchActivity(_ctx, this, _probeManager, false);
          case TAKcreaterowcatch:
              return new CRoxieServerSkipCatchActivity(_ctx, this, _probeManager, true);
          default:
              throwUnexpected();
          }
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerCatchActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerCatchActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerPullActivity : public CRoxieServerActivity
  {
      ConstPointerArray buff;
      bool started;
      unsigned index;
  
  public:
      CRoxieServerPullActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          started = false;
          index = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          started = false;
          index = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          while (buff.isItem(index))
              ReleaseRoxieRow(buff.item(index++));
          buff.kill();
          started = false;
          index = 0;
          CRoxieServerActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (!started)
              pullInput();
          if (buff.isItem(index))
          {
              const void * next = buff.item(index++);
              if (next)
                  processed++;
              return next;
          }
          return NULL;
      }
  
  protected:
      void pullInput()
      {
          bool EOGseen = false;
          for (;;)
          {
              const void * next = inputStream->nextRow();
              buff.append(next);
              if (next == NULL)
              {
                  if (EOGseen)
                      break;
                  EOGseen = true;
              }
              else
                  EOGseen = false;
          }
          started = true;
      }
  };
  
  class CRoxieServerPullActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerPullActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerPullActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  //=================================================================================
  
  class CRoxieServerTraceActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerTraceActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorTraceArg &) basehelper),
            keepLimit(0), skip(0), sample(0)
      {
          assertex(meta.hasXML());
          traceEnabled = defaultTraceEnabled && !isBlind();
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          if (ctx)
              traceEnabled = ctx->queryOptions().traceEnabled && !isBlind();
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (traceEnabled && helper.canMatchAny())
          {
              keepLimit = helper.getKeepLimit();
              if (keepLimit==(unsigned) -1)
                  keepLimit = ctx->queryOptions().traceLimit;
              skip = helper.getSkip();
              sample = helper.getSample();
              if (sample)
                  sample--;
              name.setown(helper.getName());
              if (!name)
                  name.set("Row");
          }
          else
              keepLimit = 0;
      }
      virtual void stop()
      {
          name.clear();
          CRoxieServerActivity::stop();
      }
  
      virtual bool isPassThrough()
      {
          return true;
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          const void *row = inputStream->nextRow();
          if (row)
          {
              onTrace(row);
              processed++;
          }
          return row;
      }
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          // MORE - will need rethinking once we rethink the nextRowGE interface for global smart-stepping.
          ActivityTimer t(activityStats, timeActivities);
          const void * row = inputStream->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
          if (row)
          {
              onTrace(row);
              processed++;
          }
          return row;
      }
  
      virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
      {
          return input->gatherConjunctions(collector);
      }
      virtual void resetEOF()
      {
          inputStream->resetEOF();
      }
      IInputSteppingMeta * querySteppingMeta()
      {
          return input->querySteppingMeta();
      }
  
  protected:
      void onTrace(const void *row)
      {
          if (keepLimit && helper.isValid(row))
          {
              if (skip)
                  skip--;
              else if (sample)
                  sample--;
              else
              {
                  CommonXmlWriter xmlwrite(XWFnoindent);
                  meta.toXML((const byte *) row, xmlwrite);
                  CTXLOG("TRACE: <%s>%s<%s>", name.get(), xmlwrite.str(), name.get());
                  keepLimit--;
                  sample = helper.getSample();
                  if (sample)
                      sample--;
              }
          }
      }
      OwnedRoxieString name;
      IHThorTraceArg &helper;
      unsigned keepLimit;
      unsigned skip;
      unsigned sample;
      bool traceEnabled;
  };
  
  class CRoxieServerTraceActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerTraceActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerTraceActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerTraceActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerTraceActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=================================================================================
  
  class CRoxieServerCaseActivity : public CRoxieServerMultiInputBaseActivity
  {
      IHThorCaseArg &helper;
      unsigned cond;
      bool unusedStopped;
      IEngineRowStream *active;
  
  public:
      CRoxieServerCaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
          : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs), helper((IHThorCaseArg &)basehelper)
      {
          unusedStopped = false;
          cond = 0;
          active = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerMultiInputBaseActivity::start(parentExtractSize, parentExtract, paused);
          cond = helper.getBranch();
          //CHOOSE defaults to the last argument if out of range.
          if (cond >= numInputs)
              cond = numInputs - 1;
          inputArray[cond]->start(parentExtractSize, parentExtract, paused);
          startJunction(junctionArray[cond]);
          assertex(numInputs==numStreams);
          for (unsigned idx = 0; idx < numStreams; idx++)
          {
              if (idx!=cond)
                  streamArray[idx]->stop(); // Note: stopping unused branches early helps us avoid buffering splits too long.
          }
          active = streamArray[cond];
          unusedStopped = true;
      }
  
      virtual void stop()
      {
          for (unsigned idx = 0; idx < numStreams; idx++)
          {
              if (idx==cond || !unusedStopped)
                  streamArray[idx]->stop();
          }
          CRoxieServerMultiInputBaseActivity::stop();
      }
  
      virtual void reset()
      {
          unusedStopped = false;
          active = NULL;
          CRoxieServerMultiInputBaseActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (active)
          {
              const void *ret = active->nextRow();
              if (ret)
                  processed++;
              return ret;
          }
          return NULL;
      }
  };
  
  class CRoxieServerCaseActivityFactory : public CRoxieServerMultiInputFactory
  {
      bool graphInvariant;
  public:
      CRoxieServerCaseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _graphInvariant)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          graphInvariant = _graphInvariant;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerCaseActivity(_ctx, this, _probeManager, numInputs());
      }
  
      virtual bool isGraphInvariant() const
      {
          return graphInvariant;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerCaseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _graphInvariant)
  {
      return new CRoxieServerCaseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _graphInvariant);
  }
  
  //=================================================================================
  
  class CRoxieServerIfActivity : public CRoxieServerActivity
  {
      IHThorIfArg &helper;
      IFinalRoxieInput *inputTrue;
      IFinalRoxieInput *inputFalse;
      IEngineRowStream *streamTrue;
      IEngineRowStream *streamFalse;
      unsigned sourceIdxFalse = 0;
      unsigned sourceIdxTrue = 0;
      Owned<IStrandJunction> junctionTrue;
      Owned<IStrandJunction> junctionFalse;
      bool cond;
      bool unusedStopped;
  
  public:
      CRoxieServerIfActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorIfArg &)basehelper)
      {
          inputFalse = NULL;
          inputTrue = NULL;
          streamTrue = NULL;
          streamFalse = NULL;
          unusedStopped = false;
          cond = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          cond = helper.getCondition();
          if (traceStartStop)
              DBGLOG("IfActivity::start %d - cond = %d", activityId, (int) cond);
          if (cond)
          {
              if (inputTrue)
              {
                  inputTrue->start(parentExtractSize, parentExtract, paused);
                  startJunction(junctionTrue);
              }
              if (streamFalse)
                  streamFalse->stop(); // Note: stopping unused branches early helps us avoid buffering splits too long.
          }
          else 
          {
              if (inputFalse)
              {
                  inputFalse->start(parentExtractSize, parentExtract, paused);
                  startJunction(junctionFalse);
              }
              if (streamTrue)
                  streamTrue->stop();
          }
          unusedStopped = true;
  
      }
  
      virtual void stop()
      {
          if (streamTrue && (!unusedStopped || cond))
              streamTrue->stop();
          if (streamFalse && (!unusedStopped || !cond))
              streamFalse->stop();
          CRoxieServerActivity::stop();
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 localCycles = activityStats.totalCycles;
          if (inputTrue) // if conditional input cannot assume inputTrue
              localCycles -= inputTrue->queryTotalCycles();
          if (inputFalse)
              localCycles -= inputFalse->queryTotalCycles();
          if (localCycles < 0) 
              localCycles = 0;
          return localCycles;
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          switch (idx)
          {
          case 0:
              return inputTrue;
          case 1:
              return inputFalse;
          default:
              return NULL;
          }
      }
  
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { return nullptr; }
  
      virtual IIndexReadActivityInfo *queryIndexReadActivity()
      {
          IFinalRoxieInput *in = cond ? inputTrue  : inputFalse;
          if (in)
              return in->queryIndexReadActivity();
          return NULL;
      }
  
      virtual void reset()
      {
          CRoxieServerActivity::reset();
          if (inputTrue)
              inputTrue->reset();
          if (inputFalse)
              inputFalse->reset();
          resetJunction(junctionTrue);
          resetJunction(junctionFalse);
          unusedStopped = false;
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (idx==1)
          {
              inputFalse = _in;
              sourceIdxFalse = _sourceIdx;
          }
          else
          {
              assertex(!idx);
              inputTrue = _in;
              sourceIdxTrue = _sourceIdx;
          }
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          //It should be possible to stream IF(), but the number of strands would need to match on the two branches.
          streamTrue = connectSingleStream(ctx, inputTrue, sourceIdxTrue, junctionTrue, consumerOrdered);
          streamFalse = connectSingleStream(ctx, inputFalse, sourceIdxFalse, junctionFalse, consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, nullptr, consumerOrdered, nullptr);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          IEngineRowStream *in = cond ? streamTrue  : streamFalse;
          if (in)
          {
              const void * ret;
              if ((ret = in->nextRow()) != NULL)
                  processed++;
              return ret;
          }
          return NULL;
      }
  };
  
  class CRoxieServerIfActivityFactory : public CRoxieServerActivityFactory
  {
      unsigned input2;
      unsigned input2idx;
      bool graphInvariant;
  
  public:
      CRoxieServerIfActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _graphInvariant)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
          graphInvariant = _graphInvariant;
          input2 = (unsigned)-1;
          input2idx = (unsigned)-1;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerIfActivity(_ctx, this, _probeManager);
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          if (idx==1)
          {
              input2 = source;
              input2idx = sourceidx;
          }
          else
              CRoxieServerActivityFactory::setInput(idx, source, sourceidx);
      }
  
      virtual unsigned getInput(unsigned idx, unsigned &sourceidx) const
      {
          switch (idx)
          {
          case 1:
              sourceidx = input2idx;
              return input2;
          case 0:
              return CRoxieServerActivityFactory::getInput(idx, sourceidx);
          default:
              return (unsigned) -1; 
          }
      }
  
      virtual unsigned numInputs() const { return (input2 == (unsigned)-1) ? 1 : 2; }
  
      virtual bool isGraphInvariant() const
      {
          return graphInvariant;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIfActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _graphInvariant)
  {
      return new CRoxieServerIfActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _graphInvariant);
  }
  
  //=================================================================================
  
  class CRoxieServerActionBaseActivity : public CRoxieServerActivity
  {
      CriticalSection ecrit;
      Owned<IException> exception;
      bool executed;
  
  public:
      CRoxieServerActionBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          executed = false;
      }
  
      virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract) = 0; 
  
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract) 
      {
          CriticalBlock b(ecrit);
          if (exception)
              throw(exception.getLink());
          if (!executed)
          {
              try
              {
                  executed = true;
                  start(parentExtractSize, parentExtract, false);
                  doExecuteAction(parentExtractSize, parentExtract);
                  stop();
              }
              catch (IException * E)
              {
                  ctx->notifyAbort(E);
                  abort();
                  exception.set(E);
                  throw;
              }
          }
      }
  
      virtual void reset()
      {
          executed = false;
          exception.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          return NULL;
      }
  
  };
  
  class CRoxieServerIfActionActivity : public CRoxieServerActionBaseActivity
  {
      IHThorIfArg &helper;
  
  public:
      CRoxieServerIfActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActionBaseActivity(_ctx, _factory, _probeManager), helper((IHThorIfArg &)basehelper)
      {
      }
  
      virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract) override
      {
          bool cond;
          {
              ActivityTimer t(activityStats, timeActivities);
              cond = helper.getCondition();
          }
          stopDependencies(parentExtractSize, parentExtract, cond ? 2 : 1);
          executeDependencies(parentExtractSize, parentExtract, cond ? 1 : 2);
      }
  
      virtual void stop() override
      {
          if (state != STATEstopped)
          {
              ForEachItemIn(idx, dependencies)
              {
                  if (dependencyControlIds.item(idx) != 0)
                      dependencies.item(idx).stop();
              }
          }
          CRoxieServerActionBaseActivity::stop();
      }
  
  };
  
  class CRoxieServerIfActionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerIfActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerIfActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIfActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerIfActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerParallelActionActivity : public CRoxieServerActionBaseActivity
  {
  public:
      CRoxieServerParallelActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActionBaseActivity(_ctx, _factory, _probeManager)
      {
      }
  
      virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract) 
      {
  #ifdef PARALLEL_EXECUTE
          CParallelActivityExecutor afor(dependencies, parentExtractSize, parentExtract);
          afor.For(dependencies.ordinality(), dependencies.ordinality(), true);
  #else
          ForEachItemIn(idx, dependencies)
          {
              dependencies.item(idx).execute(parentExtractSize, parentExtract);
          }
  #endif
      }
  
  };
  
  class CRoxieServerParallelActionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerParallelActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
          assertex(!isRoot);      // non-internal should be expanded out..
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerParallelActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerParallelActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerParallelActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerSequentialActionActivity : public CRoxieServerActionBaseActivity
  {
      IHThorSequentialArg &helper;
  
  public:
      CRoxieServerSequentialActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActionBaseActivity(_ctx, _factory, _probeManager), helper((IHThorSequentialArg &)basehelper)
      {
      }
  
      virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract) 
      {
          unsigned numBranches = helper.numBranches();
          try
          {
              for (unsigned branch=1; branch <= numBranches; branch++)
                  executeDependencies(parentExtractSize, parentExtract, branch);
          }
          catch (...)
          {
              for (unsigned branch=1; branch <= numBranches; branch++)
                  stopDependencies(parentExtractSize, parentExtract, branch);
              throw;
          }
      }
  
  };
  
  class CRoxieServerSequentialActionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerSequentialActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSequentialActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSequentialActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerSequentialActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerWhenActivity : public CRoxieServerActivity
  {
  public:
      CRoxieServerWhenActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          savedExtractSize = 0;
          savedExtract = NULL;
          eofseen = false;
          eogseen = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          savedExtractSize = parentExtractSize;
          savedExtract = parentExtract;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          executeDependencies(parentExtractSize, parentExtract, WhenBeforeId);
          executeDependencies(parentExtractSize, parentExtract, WhenParallelId);        // MORE: This should probably be done in parallel!
          eofseen = false;
          eogseen = false;
      }
  
      virtual void stop()
      {
          if (state == STATEreset || (!aborted && !eofseen))  // If someone else aborted, then WHEN clauses don't trigger, whatever
          {
              stopDependencies(savedExtractSize, savedExtract, WhenSuccessId);
              stopDependencies(savedExtractSize, savedExtract, WhenFailureId);
          }
          else if (state != STATEstopped)
          {
              stopDependencies(savedExtractSize, savedExtract, aborted ? WhenSuccessId : WhenFailureId);  // These ones don't get executed
              executeDependencies(savedExtractSize, savedExtract, aborted ? WhenFailureId : WhenSuccessId); // These ones do
          }
          CRoxieServerActivity::stop();
      }
  
      virtual const void *nextRow()
      {
          try
          {
              ActivityTimer t(activityStats, timeActivities); // bit of a waste of time....
              const void *ret = inputStream->nextRow();
              if (!ret)
              {
                  if (eogseen)
                      eofseen = true;
                  else
                      eogseen = true;
              }
              else
                  eogseen = false;
              return ret;
          }
          catch (...)
          {
              aborted = true;
              throw;
          }
      }
  
  protected:
      const byte *savedExtract;
      unsigned savedExtractSize;
      bool eofseen;
      bool eogseen;
  };
  
  
  class CRoxieServerWhenActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerWhenActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerWhenActivity(_ctx, this, _probeManager);
      }
  };
  
  
  extern IRoxieServerActivityFactory *createRoxieServerWhenActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerWhenActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  
  //=================================================================================
  
  class CRoxieServerWhenActionActivity : public CRoxieServerActionBaseActivity
  {
  public:
      CRoxieServerWhenActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActionBaseActivity(_ctx, _factory, _probeManager)
      {
          savedExtractSize = 0;
          savedExtract = NULL;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          savedExtractSize = parentExtractSize;
          savedExtract = parentExtract;
          CRoxieServerActionBaseActivity::start(parentExtractSize, parentExtract, paused);
          executeDependencies(parentExtractSize, parentExtract, WhenBeforeId);
          executeDependencies(parentExtractSize, parentExtract, WhenParallelId);        // MORE: This should probably be done in parallel!
      }
  
      virtual void stop()
      {
          if (state != STATEstopped)
          {
              stopDependencies(savedExtractSize, savedExtract, aborted ? WhenSuccessId : WhenFailureId);  // these are NOT going to execute
              executeDependencies(savedExtractSize, savedExtract, aborted ? WhenFailureId : WhenSuccessId);
          }
          CRoxieServerActionBaseActivity::stop();
      }
  
      virtual void doExecuteAction(unsigned parentExtractSize, const byte * parentExtract)
      {
          executeDependencies(parentExtractSize, parentExtract, 1);
      }
  
  
  protected:
      unsigned savedExtractSize;
      const byte *savedExtract;
  };
  
  
  class CRoxieServerWhenActionActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerWhenActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerWhenActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  private:
      bool isRoot;
  };
  
  extern IRoxieServerActivityFactory *createRoxieServerWhenActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerWhenActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=================================================================================
  
  
  class CRoxieServerStrandedParseActivity : public CRoxieServerStrandedActivity
  {
      INlpParseAlgorithm * algorithm;
  
      class ParseProcessor : public StrandProcessor, implements IMatchedAction
      {
      protected:
          IHThorParseArg &helper;
          INlpParser * parser;
          INlpResultIterator * rowIter;
          const void * in;
          char * curSearchText;
          size32_t curSearchTextLen;
  
          bool processRecord(const void * inRec)
          {
              if (helper.searchTextNeedsFree())
                  rtlFree(curSearchText);
  
              curSearchTextLen = 0;
              curSearchText = NULL;
              helper.getSearchText(curSearchTextLen, curSearchText, inRec);
  
              return parser->performMatch(*this, in, curSearchTextLen, curSearchText);
          }
  
      public:
          ParseProcessor(CRoxieServerActivity &_parent, IEngineRowStream *_inputStream, IHThorParseArg &_helper, INlpParseAlgorithm * _algorithm)
              : StrandProcessor(_parent, _inputStream, false), helper(_helper)
          {
              parser = _algorithm->createParser(parent.queryCodeContext(), parent.queryId(), helper.queryHelper(), &helper);
              rowIter = parser->queryResultIter();
              in = NULL;
              curSearchText = NULL;
              curSearchTextLen = 0;
          }
          ~ParseProcessor()
          {
              ::Release(parser);
          }
          virtual void start()
          {
              StrandProcessor::start();
              numProcessedLastGroup = 0;
              curSearchTextLen = 0;
              curSearchText = NULL;
              in = NULL;
              parser->reset();
          }
          virtual void reset()
          {
              if (helper.searchTextNeedsFree())
                  rtlFree(curSearchText);
              curSearchText = NULL;
              ReleaseClearRoxieRow(in);
              StrandProcessor::reset();
          }
          virtual const void * nextRow()
          {
              ActivityTimer t(activityStats, timeActivities);
              for (;;)
              {
                  if (rowIter->isValid())
                  {
                      OwnedConstRoxieRow out = rowIter->getRow();
                      rowIter->next();
                      processed++;
                      return out.getClear();
                  }
  
                  ReleaseClearRoxieRow(in);
                  in = inputStream->nextRow();
                  if (!in)
                  {
                      if (processed == numProcessedLastGroup)
                          in = inputStream->nextRow();
                      if (!in)
                      {
                          processed = numProcessedLastGroup;
                          return NULL;
                      }
  
                      in = inputStream->nextRow();
                      if (!in)
                          return NULL;
                  }
  
                  processRecord(in);
                  rowIter->first();
              }
          }
          virtual unsigned onMatch(ARowBuilder & self, const void * curRecord, IMatchedResults * results, IMatchWalker * walker)
          {
              try
              {
                  return helper.transform(self, curRecord, results, walker);
              }
              catch (IException *E)
              {
                  throw parent.makeWrappedException(E);
              }
          }
      };
  
  public:
      CRoxieServerStrandedParseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, INlpParseAlgorithm * _algorithm, const StrandOptions &_strandOptions)
          : CRoxieServerStrandedActivity(_ctx, _factory, _probeManager, _strandOptions), algorithm(_algorithm)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerStrandedActivity::start(parentExtractSize, parentExtract, paused);
          onStartStrands();
      }
  
      virtual StrandProcessor *createStrandProcessor(IEngineRowStream *instream)
      {
          return new ParseProcessor(*this, instream, (IHThorParseArg &) basehelper, algorithm);
      }
      virtual StrandProcessor *createStrandSourceProcessor(bool inputOrdered) { throwUnexpected(); }
  };
  
  
  class CRoxieServerParseActivityFactory : public CRoxieServerActivityFactory
  {
      Owned<INlpParseAlgorithm> algorithm;
      Owned<IHThorParseArg> helper;
      StrandOptions strandOptions;
  
  public:
      CRoxieServerParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, IResourceContext *rc)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics), strandOptions(_graphNode)
      {
          helper.setown((IHThorParseArg *) helperFactory());
          algorithm.setown(createThorParser(rc, *helper));
          optStableInput = false;
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerStrandedParseActivity(_ctx, this, _probeManager, algorithm, strandOptions);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, IResourceContext *rc)
  {
      return new CRoxieServerParseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, rc);
  }
  
  //=====================================================================================================
  
  class CRoxieServerWorkUnitWriteActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorWorkUnitWriteArg &helper;
      bool isReread;
      bool grouped;
      IRoxieServerContext *serverContext;
      int sequence;
  
  public:
      CRoxieServerWorkUnitWriteActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _isReread, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorWorkUnitWriteArg &)basehelper), isReread(_isReread)
      {
          grouped = (helper.getFlags() & POFgrouped) != 0;
          serverContext = ctx->queryServerContext();
          if (!serverContext)
          {
              throw MakeStringException(ROXIE_PIPE_ERROR, "Workunit output activity cannot be executed in agent context");
          }
          sequence = helper.getSequence();
          if (sequence >= LibraryBaseSequence)
          {
              IConstWorkUnit *workunit = serverContext->queryQueryFactory()->queryWorkUnit();
              assertex(workunit);
              const char *storedName = helper.queryName();
              assertex(storedName);
              Owned<IConstWUResult> queryRes =  workunit->getQueryResultByName(storedName);
              if (!queryRes)
                  throw makeStringExceptionV(0, "Library cannot write to result %s that does not exist in calling query", storedName);
              Owned<IConstWUResult> libraryRes =  factory->queryQueryFactory().queryWorkUnit()->getResultBySequence(sequence);
              if (libraryRes) // Should always be present, but rather than assert, just ignore if not
              {
                  SCMStringBuffer queryFormat;
                  SCMStringBuffer libraryFormat;
                  queryRes->getResultEclSchema(queryFormat);
                  libraryRes->getResultEclSchema(libraryFormat);
                  if (!streq(queryFormat.str(), libraryFormat.str()))
                  {
                      DBGLOG("Query format: %s", queryFormat.str());
                      DBGLOG("Library format: %s", libraryFormat.str());
                      throw makeStringExceptionV(0, "Library cannot write to result %s: result type in query does not match", storedName);
                  }
              }
              sequence = queryRes->getResultSequence();
          }
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onExecute()
      {
          MemoryBuffer result;
          bool saveInContext = (int) sequence < 0 || isReread;
          if (!meta.queryOriginal()) // this is a bit of a hack - don't know why no meta on an output....
              meta.set(input->queryOutputMeta());
          Owned<IOutputRowSerializer> rowSerializer;
          Owned<IXmlWriter> xmlwriter;
          bool appendRaw = false;
  
          IHpccProtocolResultsWriter *results = NULL;
          IHpccNativeProtocolResultsWriter *nativeResults = NULL;
          IHpccProtocolResponse *protocol = serverContext->queryProtocol();
          unsigned protocolFlags = (protocol) ? protocol->getFlags() : 0;
          if ((int) sequence >= 0)
          {
              results = (protocol) ? protocol->queryHpccResultsSection() : NULL;
              if (results)
              {
                  nativeResults = dynamic_cast<IHpccNativeProtocolResultsWriter*>(results);
                  xmlwriter.setown(results->addDataset(helper.queryName(), sequence, "Dataset", appendRaw, serverContext->getXmlFlags(), (helper.getFlags() & POFextend) != 0, serverContext->queryXmlns(sequence)));  //xmlwriter only returned if needed
              }
          }
          size32_t outputLimitBytes = 0;
          IConstWorkUnit *workunit = sequence == ResultSequenceInternal ? nullptr : serverContext->queryWorkUnit();
          if (workunit)
          {
              size32_t outputLimit;
              if (helper.getFlags() & POFmaxsize)
                  outputLimit = helper.getMaxSize();
              else
              {
                  // In absense of OPT_OUTPUTLIMIT check pre 5.2 legacy name OPT_OUTPUTLIMIT_LEGACY
                  outputLimit = workunit->getDebugValueInt(OPT_OUTPUTLIMIT, workunit->getDebugValueInt(OPT_OUTPUTLIMIT_LEGACY, defaultDaliResultLimit));
              }
              if (outputLimit>daliResultOutputMax)
                  throw MakeStringException(0, "Dali result outputs are restricted to a maximum of %d MB, the current limit is %d MB. A huge dali result usually indicates the ECL needs altering.", daliResultOutputMax, defaultDaliResultLimit);
              assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
              outputLimitBytes = outputLimit * 0x100000;
          }
          if (workunit != NULL || (results && (protocol->getFlags() & HPCC_PROTOCOL_NATIVE_RAW)))
          {
              ensureRowAllocator();
              rowSerializer.setown(rowAllocator->createDiskSerializer(ctx->queryCodeContext()));
          }
          __int64 initialProcessed = processed;
          RtlLinkedDatasetBuilder builder(rowAllocator);
          for (;;)
          {
              const void *row = inputStream->nextRow();
              if (saveInContext)
              {
                  if (row || grouped)
                      builder.append(row);
              }
              if (grouped && (processed != initialProcessed))
              {
                  if (workunit)
                      result.append(row == NULL);
                  if (results)
                  {
                      if ((protocolFlags & HPCC_PROTOCOL_NATIVE_RAW) && nativeResults)
                      {
                          char val = (char)(row == NULL);
                          nativeResults->appendRaw(sequence, 1, &val);
                      }
                      else if (xmlwriter)
                      {
                          xmlwriter->outputBeginNested("Row", false);
                          xmlwriter->outputCString("1", "@__GroupBreak__");
                          xmlwriter->outputEndNested("Row");
                      }
                  }
              }
              if (!row)
              {
                  row = inputStream->nextRow();
                  if (!row)
                      break;
                  if (saveInContext)
                      builder.append(row);
              }
              processed++;
              if (workunit)
              {
                  CThorDemoRowSerializer serializerTarget(result);
                  rowSerializer->serialize(serializerTarget, (const byte *) row);
              }
              if ((int) sequence >= 0)
              {
                  if (appendRaw && nativeResults)
                  {
                      // MORE - should be able to serialize straight to the response...
                      MemoryBuffer rowbuff;
                      CThorDemoRowSerializer serializerTarget(rowbuff);
                      rowSerializer->serialize(serializerTarget, (const byte *) row);
                      nativeResults->appendRawRow(sequence, rowbuff.length(), rowbuff.toByteArray());
                  }
                  else if (xmlwriter)
                  {
                      xmlwriter->outputBeginNested(DEFAULTXMLROWTAG, false);
                      helper.serializeXml((byte *) row, *xmlwriter);
                      xmlwriter->outputEndNested(DEFAULTXMLROWTAG);
                      results->finalizeXmlRow(sequence);
                  }
                  else if (nativeResults)
                  {
                      SimpleOutputWriter x;
                      helper.serializeXml((byte *) row, x);
                      x.newline();
                      nativeResults->appendSimpleRow(sequence, x.str());
                  }
              }
              ReleaseRoxieRow(row);
              if (outputLimitBytes && result.length() > outputLimitBytes)
              {
                  StringBuffer errMsg("Dataset too large to output to workunit (limit ");
                  errMsg.append(outputLimitBytes/0x100000).append(" megabytes), in result (");
                  const char *name = helper.queryName();
                  if (name)
                      errMsg.append("name=").append(name);
                  else
                      errMsg.append("sequence=").append(helper.getSequence());
                  errMsg.append(")");
                  throw MakeStringExceptionDirect(0, errMsg.str());
              }
          }
          if (xmlwriter)
              xmlwriter->outputEndArray(DEFAULTXMLROWTAG);
          const char *storedName = helper.queryName();
          if (!storedName)
              storedName = "Dataset";
          if (saveInContext)
              serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
          if (workunit)
              serverContext->appendResultRawContext(storedName, sequence, result.length(), result.toByteArray(), processed, (helper.getFlags() & POFextend) != 0, false); // MORE - shame to do extra copy...
      }
  };
  
  class CRoxieServerWorkUnitWriteActivityFactory : public CRoxieServerInternalSinkFactory
  {
      bool isReread;
      bool isUnused = false;
  
  public:
      CRoxieServerWorkUnitWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot)
      {
          isReread = usageCount > 0;
          Owned<IHThorWorkUnitWriteArg> helper = (IHThorWorkUnitWriteArg *) helperFactory();
          isInternal = (helper->getSequence()==ResultSequenceInternal);
          isUnused = isInternal && (usageCount == 0);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (isUnused && !CRoxieServerInternalSinkFactory::isSink())
          {
              if (_ctx->queryTraceLevel() > 2)
                  DBGLOG("Workunit write %u is unused - create null activity", id);
              //Create a null sink activity that is always executed to ensure that stop() is called on the input.
              return createRoxieServerNullSinkActivity(_ctx, this, _probeManager);
          }
  
          return new CRoxieServerWorkUnitWriteActivity(_ctx, this, _probeManager, isReread, usageCount);
      }
  
      virtual bool isSink() const
      {
          return isUnused || CRoxieServerInternalSinkFactory::isSink();
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
  {
      return new CRoxieServerWorkUnitWriteActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _isRoot);
  
  }
  
  //=====================================================================================================
  
  class CRoxieServerWorkUnitWriteDictActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorDictionaryWorkUnitWriteArg &helper;
      IRoxieServerContext *serverContext;
  
  public:
      CRoxieServerWorkUnitWriteDictActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _usageCount)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _usageCount), helper((IHThorDictionaryWorkUnitWriteArg &)basehelper)
      {
          serverContext = NULL;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
          serverContext = ctx->queryServerContext();
          if (!serverContext)
          {
              throw MakeStringException(ROXIE_PIPE_ERROR, "Write Dictionary activity cannot be executed in agent context");
          }
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void onExecute()
      {
          int sequence = helper.getSequence();
          const char *storedName = helper.queryName();
          assertex(storedName && *storedName);
          assertex(sequence < 0);
  
          RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
          for (;;)
          {
              const void *row = inputStream->ungroupedNextRow();
              if (!row)
                  break;
              builder.appendOwn(row);
              processed++;
          }
          serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
      }
  };
  
  class CRoxieServerWorkUnitWriteDictActivityFactory : public CRoxieServerInternalSinkFactory
  {
  
  public:
      CRoxieServerWorkUnitWriteDictActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot)
      {
          Owned<IHThorDictionaryWorkUnitWriteArg> helper = (IHThorDictionaryWorkUnitWriteArg *) helperFactory();
          isInternal = (helper->getSequence()==ResultSequenceInternal);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerWorkUnitWriteDictActivity(_ctx, this, _probeManager, usageCount);
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteDictActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
  {
      return new CRoxieServerWorkUnitWriteDictActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _isRoot);
  
  }
  
  //=================================================================================
  
  class CRoxieServerRemoteResultActivity : public CRoxieServerInternalSinkActivity
  {
      IHThorRemoteResultArg &helper;
  
  public:
      CRoxieServerRemoteResultActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numOutputs)
          : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorRemoteResultArg &)basehelper)
      {
      }
  
      virtual void onExecute() 
      {
          OwnedConstRoxieRow row = inputStream->nextRow();
          helper.sendResult(row);  // should be only one row or something has gone wrong!
      }
  
  };
  
  class CRoxieServerRemoteResultActivityFactory : public CRoxieServerInternalSinkFactory
  {
  public:
      CRoxieServerRemoteResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
          : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics, _usageCount, _isRoot)
      {
          Owned<IHThorRemoteResultArg> helper = (IHThorRemoteResultArg *) helperFactory();
          isInternal = (helper->getSequence()==ResultSequenceInternal);
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (dependentCount==0 && !CRoxieServerInternalSinkFactory::isSink())
              return new CRoxieServerNullSinkActivity(_ctx, this, _probeManager);
          else
              return new CRoxieServerRemoteResultActivity(_ctx, this, _probeManager, usageCount);
      }
  
      virtual bool isSink() const override
      {
          return CRoxieServerInternalSinkFactory::isSink() || dependentCount == 0;  // Codegen normally optimizes these away, but if it doesn't we need to treat as a (null) sink rather than a dependency or upstream activities are not stopped properly
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerRemoteResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, unsigned _usageCount, bool _isRoot)
  {
      return new CRoxieServerRemoteResultActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _usageCount, _isRoot);
  }
  
  //=================================================================================
  
  class CRoxieServerXmlParseActivity : public CRoxieServerActivity, implements IXMLSelect
  {
      IHThorXmlParseArg &helper;
  
      Owned<IXMLParse> xmlParser;
      const void * in;
      char * srchStr;
      unsigned numProcessedLastGroup;
      bool srchStrNeedsFree;
      Owned<IColumnProvider> lastMatch;
  
  public:
      IMPLEMENT_IINTERFACE_USING(CRoxieServerActivity)
  
      CRoxieServerXmlParseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
          helper((IHThorXmlParseArg &)basehelper)
      {
          srchStrNeedsFree = helper.searchTextNeedsFree();
          numProcessedLastGroup = 0;
          srchStr = NULL;
          in = NULL;
      }
  
      ~CRoxieServerXmlParseActivity()
      {
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          numProcessedLastGroup = 0;
          srchStr = NULL;
          in = NULL;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          if (helper.searchTextNeedsFree())
              rtlFree(srchStr);
          srchStr = NULL;
          ReleaseClearRoxieRow(in);
          xmlParser.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset)
      {
          lastMatch.set(&entry);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if(xmlParser)
              {
                  for (;;)
                  {
                      if(!xmlParser->next())
                      {
                          if (srchStrNeedsFree)
                          {
                              rtlFree(srchStr);
                              srchStr = NULL;
                          }
                          xmlParser.clear();
                          break;
                      }
                      if(lastMatch)
                      {
                          try
                          {
                              RtlDynamicRowBuilder rowBuilder(rowAllocator);
                              size32_t outSize = helper.transform(rowBuilder, in, lastMatch);
                              lastMatch.clear();
                              if (outSize)
                              {
                                  processed++;
                                  return rowBuilder.finalizeRowClear(outSize);
                              }
                          }
                          catch (IException *E)
                          {
                              throw makeWrappedException(E);
                          }
                      }
                  }
              }
              ReleaseClearRoxieRow(in);
              in = inputStream->nextRow();
              if(!in)
              {
                  if(numProcessedLastGroup == processed)
                      in = inputStream->nextRow();
                  if(!in)
                  {
                      numProcessedLastGroup = processed;
                      return NULL;
                  }
              }
              size32_t srchLen;
              helper.getSearchText(srchLen, srchStr, in);
              OwnedRoxieString xmlIteratorPath(helper.getXmlIteratorPath());
              xmlParser.setown(createXMLParse(srchStr, srchLen, xmlIteratorPath, *this));
          }   
      }
  
  };
  
  class CRoxieServerXmlParseActivityFactory : public CRoxieServerActivityFactory
  {
  
  public:
      CRoxieServerXmlParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, actStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerXmlParseActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerXmlParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerXmlParseActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //=====================================================================================================
  
  class CRoxieServerDiskReadBaseActivity : public CRoxieServerActivity, implements IRoxieServerErrorHandler, implements IIndexReadContext
  {
  protected:
      IHThorDiskReadBaseArg &helper;
      IHThorSourceLimitTransformExtra * limitTransformExtra = nullptr;
      IHThorCompoundExtra * compoundHelper;
      RemoteActivityId remoteId;              // Note we copy it rather than reference
      Owned<CSkippableRemoteResultAdaptor> remote;
      unsigned numParts;
      unsigned __int64 rowLimit;
      unsigned __int64 stopAfter;
      Linked<IInMemoryIndexManager> manager;
      Linked<ITranslatorSet> translators;
      ScoredRowFilter postFilter;
      Owned<IDirectReader> reader;
      bool eof;
      bool variableFileName;
      bool isOpt;
      bool sorted;
      bool maySkip;
      bool isLocal;
      bool forceRemote;
      bool isGrouped = false;
      CachedOutputMetaData diskSize;
      Owned<const IResolvedFile> varFileInfo;
  
      inline bool useRemote()
      {
          return remote != NULL && (forceRemote || numParts > 1);
      }
  
  public:
      IMPLEMENT_IINTERFACE_USING(CRoxieServerActivity)
  
      CRoxieServerDiskReadBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
                                       unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            helper((IHThorDiskReadBaseArg &)basehelper),
            remoteId(_remoteId),
            numParts(_numParts),
            manager(_manager),
            translators(_translators),
            sorted(_sorted),
            maySkip(_maySkip),
            isLocal(_isLocal)
      {
          forceRemote = factory->queryQueryFactory().queryOptions().disableLocalOptimizations;
          if ((forceRemote || numParts != 1) && !isLocal)  // NOTE : when numParts == 0 (variable case) we create, even though we may not use
              remote.setown(new CSkippableRemoteResultAdaptor(_ctx, this, remoteId, meta.queryOriginal(), helper, *this, sorted, false, _maySkip));
          compoundHelper = NULL;
          eof = false;
          rowLimit = (unsigned __int64) -1;
          stopAfter = I64C(0x7FFFFFFFFFFFFFFF);
          diskSize.set(helper.queryDiskRecordSize()->querySerializedDiskMeta());
          isGrouped = diskSize.isGrouped();
          variableFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
          isOpt = (helper.getFlags() & TDRoptional) != 0;
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          if (remote)
              remote->onCreate(_colocalParent);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (compoundHelper)
          {
              rowLimit = compoundHelper->getRowLimit();
              stopAfter = compoundHelper->getChooseNLimit();
          }
          if (!helper.canMatchAny())
              eof = true;
          else
          {
              if (variableFileName)
              {
                  OwnedRoxieString fileName(helper.getFileName());
                  varFileInfo.setown(resolveLFNFlat(fileName, isOpt, factory->isActivityCodeSigned()));
                  numParts = 0;
                  if (varFileInfo)
                      numParts = varFileInfo->getNumParts();
              }
              if (!numParts)
              {
                  eof = true;
              }
              else if (useRemote())
              {
                  remote->onStart(parentExtractSize, parentExtract);
                  remote->setLimits(rowLimit, (unsigned __int64) -1, stopAfter);
                  unsigned fileNo = 0;
                  // Translation into a message per channel done elsewhere....
                  remote->getMem(0, fileNo, 0);
                  remote->flush();
                  remote->senddone();
              }
              else
              {
                  if (variableFileName)
                  {
                      unsigned channel = isLocal ? factory->queryQueryFactory().queryChannel() : 0;
                      unsigned expectedFormatCrc = helper.getDiskFormatCrc();
                      unsigned projectedFormatCrc = helper.getProjectedFormatCrc();
                      translators.setown(varFileInfo->getTranslators(projectedFormatCrc, helper.queryProjectedDiskRecordSize(), expectedFormatCrc, helper.queryDiskRecordSize(), getEnableFieldTranslation(), FileFormatMode::flat, queryFactory()->queryQueryFactory().queryQueryName()));
                      manager.setown(varFileInfo->getIndexManager(isOpt, channel, translators->queryActualLayout(0), false));
                  }
                  assertex(manager != NULL);
                  postFilter.clear();
                  helper.createSegmentMonitors(this);
                  const IKeyTranslator *keyTranslator = translators->queryKeyTranslator(0);  // any part would do - in-memory requires all actuals to have same layout
                  if (keyTranslator)
                      keyTranslator->translate(postFilter);
                  reader.setown(manager->selectKey(postFilter, translators, *this));
                  if (!reader)
                      reader.setown(manager->createReader(postFilter, isGrouped, 0, 0, 1, translators));
                  assertex(reader);
                  helper.setCallback(reader);
              }
          }
      }
  
      virtual void append(IKeySegmentMonitor *segment)
      {
          segment->Release();
          throwUnexpected();
      }
  
      virtual void append(FFoption option, const IFieldFilter * filter)
      {
          if (filter->isWild())
              filter->Release();
          else
              postFilter.addFilter(*filter);
      }
  
      virtual void stop()
      {
          if (useRemote())
              remote->onStop();
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          if (useRemote())
          {
              processed = remote->processed;
              remote->processed = 0;
              remote->onReset();
          }
          varFileInfo.clear();
          eof = false;
          reader.clear();
          CRoxieServerActivity::reset();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          assertex(compoundHelper);
          if (isKeyed) // MORE does this exist for diskread? should it?
          {
              if (helper.getFlags() & (TDRkeyedlimitskips|TDRkeyedlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(true);
              }
              else
              {
                  UNIMPLEMENTED;
                  //compoundHelper->onKeyedLimitExceeded(); Doesn't exist - though the flags do... interesting...
              }
          }
          else
          {
              if (helper.getFlags() & (TDRlimitskips|TDRlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(false);
              }
              else
                  compoundHelper->onLimitExceeded();
          }
      }
  
      virtual const void * createLimitFailRow(bool isKeyed)
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          assertex(limitTransformExtra);
          size32_t outSize = isKeyed ? limitTransformExtra->transformOnKeyedLimitExceeded(rowBuilder) : limitTransformExtra->transformOnLimitExceeded(rowBuilder);
          if (outSize)
              return rowBuilder.finalizeRowClear(outSize);
          return NULL;
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          if (remote)
              remote->gatherStats(merged);
      }
  };
  
  class CRoxieServerDiskReadActivity : public CRoxieServerDiskReadBaseActivity
  {
      IHThorDiskReadArg * readHelper;
      ConstPointerArray readrows;
      bool readAheadDone;
      unsigned readIndex;
      unsigned lastGroupProcessed;
      bool someInGroup = false;
  
  public:
      CRoxieServerDiskReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators)
      {
          compoundHelper = (IHThorDiskReadArg *)&helper;
          readHelper = (IHThorDiskReadArg *)&helper;
          limitTransformExtra = readHelper;
          readAheadDone = false;
          readIndex = 0;
          lastGroupProcessed = processed;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          rowLimit = compoundHelper->getRowLimit();
          stopAfter = compoundHelper->getChooseNLimit();
          CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
          readAheadDone = false;
          readIndex = 0;
          lastGroupProcessed = processed;
          someInGroup = false;
      }
  
      virtual void reset()
      {
          roxiemem::ReleaseRoxieRowRange(readrows.getArray(), readIndex, readrows.ordinality());
          readrows.kill();
          readAheadDone = false;
          readIndex = 0;
          lastGroupProcessed = processed;
          someInGroup = false;
          CRoxieServerDiskReadBaseActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          else if (useRemote())
              return remote->nextRow();
          else if (maySkip)
          {
              if (!readAheadDone)
              {
                  unsigned preprocessed = 0;
                  while (!eof)
                  {
                      const void *row = _nextRow();
                      if (row)
                          preprocessed++;
                      if (preprocessed > rowLimit)
                      {
                          ReleaseRoxieRow(row);
                          roxiemem::ReleaseRoxieRowRange(readrows.getArray(), readIndex, readrows.ordinality());
                          readrows.kill();
                          readIndex = 0;
                          eof = true;
                          if (ctx->queryDebugContext())
                              ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                          if (helper.getFlags() & TDRlimitskips)
                              return NULL;
                          else if (helper.getFlags() & TDRlimitcreates)
                              return createLimitFailRow(false);
                          else
                              throwUnexpected();
                      }
                      if (preprocessed > stopAfter) // MORE - bit of a strange place to check
                      {
                          eof = true;
                          ReleaseRoxieRow(row);
                          break;
                      }
                      readrows.append(row);
                  }
                  readAheadDone = true;
              }
              if (readrows.isItem(readIndex))
              {
                  const void *ret = readrows.item(readIndex++);
                  if (ret)
                      processed++;
                  return ret;
              }
              else
              {
                  eof = true;
                  return NULL;
              }
          }
          else
          {
              const void *ret = _nextRow();
              if (ret)
              {
                  processed++;
                  if (processed > rowLimit)
                  {
                      if (traceLevel > 4)
                          DBGLOG("activityid = %d line = %d", activityId, __LINE__);
                      ReleaseRoxieRow(ret);
                      compoundHelper->onLimitExceeded();
                      throwUnexpected(); // onLimitExceeded is not supposed to return
                  }
                  if (processed > stopAfter) // MORE - bit of a strange place to check
                  {
                      eof = true;
                      ReleaseRoxieRow(ret);
                      return NULL;
                  }
              }
              return ret;
          }
      }
  
      const void *_nextRow()
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          unsigned transformedSize = 0;
          for (;;)
          {
              const byte *nextRec = reader->nextRow();
              if (nextRec)
              {
                  someInGroup = true;
                  if (likely(readHelper->canMatch(nextRec)))
                  {
                      transformedSize = readHelper->transform(rowBuilder, nextRec);
                      reader->finishedRow();
                      if (transformedSize)
                          break;
                  }
                  else
                      reader->finishedRow();
              }
              else if (isGrouped)
              {
                  if (lastGroupProcessed != processed)
                  {
                      someInGroup = false;
                      lastGroupProcessed = processed;
                      return nullptr;
                  }
                  else if (!someInGroup)
                  {
                      eof = true;
                      return nullptr;
                  }
              }
              else
              {
                  eof = true;
                  return nullptr;
              }
          }
          return rowBuilder.finalizeRowClear(transformedSize);
      }
  };
  
  class CRoxieServerXmlReadActivity : public CRoxieServerDiskReadBaseActivity, implements IXMLSelect, implements IThorDiskCallback
  {
      IHThorXmlReadArg * readHelper;
      Owned<IXmlToRowTransformer> rowTransformer;
      Owned<IXMLParse> xmlParser;
      Owned<IColumnProvider> lastMatch;
      unsigned __int64 fileoffset;
      IDirectStreamReader *streamReader = nullptr;
  public:
      IMPLEMENT_IINTERFACE_USING(CRoxieServerDiskReadBaseActivity)
      CRoxieServerXmlReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
                                  unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators)
      {
          compoundHelper = NULL;
          readHelper = (IHThorXmlReadArg *)&helper;
          fileoffset = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          rowLimit = readHelper->getRowLimit();
          stopAfter = readHelper->getChooseNLimit();
          CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
          if (!useRemote())
          {
              rowTransformer.set(readHelper->queryTransformer());
              assertex(reader != NULL);
              streamReader = reader->queryDirectStreamReader();
              assertex(streamReader != NULL);
              OwnedRoxieString xmlIterator(readHelper->getXmlIteratorPath());
              if (factory->getKind()==TAKjsonread)
                  xmlParser.setown(createJSONParse(*streamReader, xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
              else
                  xmlParser.setown(createXMLParse(*streamReader, xmlIterator, *this, (0 != (TDRxmlnoroot & readHelper->getFlags()))?ptr_noRoot:ptr_none, (readHelper->getFlags() & TDRusexmlcontents) != 0));
          }
      }
  
      virtual void reset()
      {
          CRoxieServerDiskReadBaseActivity::reset();
          rowTransformer.clear();
          xmlParser.clear();
      }
  
  
      virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset)
      {
          fileoffset = startOffset;
          lastMatch.set(&entry);
      }
  
      //interface IThorDiskCallback
      virtual unsigned __int64 getFilePosition(const void * row)
      {
          return fileoffset;
      }
      virtual unsigned __int64 getLocalFilePosition(const void * row)
      {
          return streamReader->makeFilePositionLocal(fileoffset);
      }
      virtual const char * queryLogicalFilename(const void * row)
      {
          return reader->queryLogicalFilename(row);
      }
      virtual const byte * lookupBlob(unsigned __int64 id) { UNIMPLEMENTED; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          else if (useRemote())
              return remote->nextRow();
          assertex(xmlParser != NULL);
          try
          {
              while (!eof)
              {
                  //call to next() will callback on the IXmlSelect interface
                  bool gotNext = false;
                  gotNext = xmlParser->next();
                  if(!gotNext)
                      eof = true;
                  else if (lastMatch)
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      unsigned sizeGot = rowTransformer->transform(rowBuilder, lastMatch, this);
                      lastMatch.clear();
                      fileoffset = 0;
                      if (sizeGot)
                      {
                          OwnedConstRoxieRow ret = rowBuilder.finalizeRowClear(sizeGot); 
                          if (processed > rowLimit)
                          {
                              if (traceLevel > 4)
                                  DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                              readHelper->onLimitExceeded();
                              throwUnexpected(); // onLimitExceeded is not supposed to return
                          }
                          processed++;
                          if (processed > stopAfter) // MORE - bit of a strange place to check
                          {
                              eof = true;
                              return NULL;
                          }
                          return ret.getClear();
                      }
                  }
              }
              return NULL;
          }
          catch(IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  };
  
  class CRoxieServerCsvReadActivity : public CRoxieServerDiskReadBaseActivity
  {
      IHThorCsvReadArg *readHelper;
      ICsvParameters * csvInfo;
      unsigned headerLines;
      unsigned maxDiskSize;
      CSVSplitter csvSplitter;    
      unsigned __int64 localOffset;
      IDirectStreamReader *streamReader = nullptr;
      const char *quotes;
      const char *separators;
      const char *terminators;
      const char *escapes;
      size32_t maxRowSize;
  public:
      CRoxieServerCsvReadActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
                                  unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager, ITranslatorSet *_translators,
                                  const char *_quotes, const char *_separators, const char *_terminators, const char *_escapes, size32_t _maxRowSize)
          : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, _maySkip, _manager, _translators),
            quotes(_quotes), separators(_separators), terminators(_terminators), escapes(_escapes), maxRowSize(_maxRowSize)
      {
          compoundHelper = NULL;
          readHelper = (IHThorCsvReadArg *)&helper;
          rowLimit = readHelper->getRowLimit();
          stopAfter = readHelper->getChooseNLimit();
          csvInfo = readHelper->queryCsvParameters();
          maxDiskSize = csvInfo->queryMaxSize();
          localOffset = 0;
          headerLines = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          rowLimit = readHelper->getRowLimit();
          stopAfter = readHelper->getChooseNLimit();
          CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
          if (!useRemote())
          {
              if (!eof)
              {
                  assertex(reader != nullptr);
                  streamReader = reader->queryDirectStreamReader();
                  assertex(streamReader != nullptr);
                  headerLines = csvInfo->queryHeaderLen();
                  if (headerLines && isLocal && streamReader->queryFilePart() != 1)
                      headerLines = 0;  // MORE - you could argue that if SINGLE not specified, should skip from all parts. But it would be painful since we have already concatenated and no-one else does...
                  if (varFileInfo)
                  {
                      const IPropertyTree *options = varFileInfo->queryProperties();
                      if (options)
                      {
                          quotes = options->queryProp("@csvQuote");
                          separators = options->queryProp("@csvSeparate");
                          terminators = options->queryProp("@csvTerminate");
                          escapes = options->queryProp("@csvEscape");
                      }
                  }
                  csvSplitter.init(readHelper->getMaxColumns(), csvInfo, quotes, separators, terminators, escapes);
              }
          }
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          else if (useRemote())
              return remote->nextRow();
          try
          {
              while (!eof)
              {
                  size32_t thisLineLength = csvSplitter.splitLine(streamReader, maxRowSize);
                  if (!thisLineLength)
                      break;
                  if (headerLines)
                  {
                      headerLines--;
                      streamReader->skip(thisLineLength);
                  }
                  else
                  {
                      RtlDynamicRowBuilder rowBuilder(rowAllocator);
                      unsigned transformedSize = readHelper->transform(rowBuilder, csvSplitter.queryLengths(), (const char * *)csvSplitter.queryData());
                      streamReader->skip(thisLineLength);
                      if (transformedSize)
                      {
                          OwnedConstRoxieRow ret = rowBuilder.finalizeRowClear(transformedSize);
                          if (processed > rowLimit)
                          {
                              readHelper->onLimitExceeded();
                              throwUnexpected(); // onLimitExceeded is not supposed to return
                          }
                          processed++;
                          if (processed > stopAfter) // MORE - bit of a strange place to check
                          {
                              eof = true;
                              return NULL;
                          }
                          return ret.getClear();
                      }
                  }
              }
              return NULL;
          }
          catch(IException *E)
          {
              throw makeWrappedException(E);
          }
      }
      virtual void reset()
      {
          CRoxieServerDiskReadBaseActivity::reset();
          csvSplitter.reset();
          localOffset = 0;
      }
  
      virtual unsigned __int64 getFilePosition(const void * row)
      {
          UNIMPLEMENTED; // we know offset in the reader but not sure it helps us much
      }
  
      virtual unsigned __int64 getLocalFilePosition(const void * row)
      {
          UNIMPLEMENTED;
      }
  
  };
  
  class CRoxieServerDiskNormalizeActivity : public CRoxieServerDiskReadBaseActivity
  {
      IHThorDiskNormalizeArg *normalizeHelper;
      bool firstPending;
  
  public:
      CRoxieServerDiskNormalizeActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _sorted, false, _manager, _translators)
      {
          compoundHelper = (IHThorDiskNormalizeArg *)&helper;
          normalizeHelper = (IHThorDiskNormalizeArg *)&helper;
          limitTransformExtra = normalizeHelper;
          firstPending = true;
      }
  
      virtual void reset()
      {
          firstPending = true;
          CRoxieServerDiskReadBaseActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          else if (useRemote())
              return remote->nextRow();
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          unsigned transformedSize = 0;
          for (;;)
          {
              while (firstPending)
              {
                  const byte *nextRec = reader->nextRow();
                  if (!nextRec)
                  {
                      eof = true;
                      return NULL;
                  }
                  if (normalizeHelper->first(nextRec))
                      firstPending = false;
                  reader->finishedRow();
              }
              transformedSize = normalizeHelper->transform(rowBuilder);
              firstPending = !normalizeHelper->next();
              if (transformedSize)
                  break;
          }
          OwnedConstRoxieRow recBuffer = rowBuilder.finalizeRowClear(transformedSize);
          processed++;
          if (processed > rowLimit)
          {
              compoundHelper->onLimitExceeded();
              throwUnexpected(); // onLimitExceeded is not supposed to return
          }
          if (processed > stopAfter) // MORE - bit of a strange place to check
          {
              eof = true;
              return NULL;
          }
          return recBuffer.getClear();
      }
  };
  
  
  class CRoxieServerDiskAggregateBaseActivity : public CRoxieServerDiskReadBaseActivity
  {
  protected:
      bool done;
  
  public:
      CRoxieServerDiskAggregateBaseActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, false, false, _manager, _translators),
            done(false)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL: this;
      }
  };
  
  class CRoxieServerDiskCountActivity : public CRoxieServerDiskAggregateBaseActivity
  {
      IHThorDiskCountArg & countHelper;
      unsigned __int64 choosenLimit;
  
      unsigned __int64 getSkippedCount()
      {
          unsigned flags = countHelper.getFlags();
          if (flags & TDRlimitskips)
              return 0;
          else if (flags & TDRlimitcreates)
              return 1;
          else
          {
              countHelper.onLimitExceeded();
              throwUnexpected(); // onLimitExceeded should always throw exception
          }
      }
  
  public:
      CRoxieServerDiskCountActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
            countHelper((IHThorDiskCountArg &)basehelper)
      {
          choosenLimit = 0;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          choosenLimit = countHelper.getChooseNLimit();
          rowLimit = countHelper.getRowLimit();
  //      keyedLimit = countHelper->getKeyedLimit(); // more - should there be one?
          CRoxieServerDiskAggregateBaseActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          done = true;
  
          unsigned __int64 totalCount = 0;
          if (helper.canMatchAny() && !eof)
          {
              if (useRemote())
              {
                  for (;;)
                  {
                      const void * next = remote->nextRow();
                      if (!next)
                          break;
                      if (meta.getFixedSize() == 1)
                          totalCount += *(byte *)next;
                      else
                          totalCount += *(unsigned __int64 *)next;
                      ReleaseRoxieRow(next);
                      if (totalCount > rowLimit)
                      {
                          totalCount = getSkippedCount();
                          break;
                      }
                      else if (totalCount >= choosenLimit)
                      {
                          totalCount = choosenLimit;
                          break;
                      }
                  }
              }
              else
              {
                  for (;;)
                  {
                      const byte *nextRec = reader->nextRow();
                      if (!nextRec)
                          break;
                      totalCount += countHelper.numValid(nextRec);
                      reader->finishedRow();
                      if (totalCount > rowLimit)
                      {
                          totalCount = getSkippedCount();
                          break;
                      }
                      else if (totalCount >= choosenLimit)
                      {
                          totalCount = choosenLimit;
                          break;
                      }
                  }
              }
          }
          processed++;
          size32_t rowSize = meta.getFixedSize();
          void * result = rowAllocator->createRow();
          if (rowSize == 1)
              *(byte *)result = (byte)totalCount;
          else
          {
              assertex(rowSize == sizeof(unsigned __int64));
              *(unsigned __int64 *)result = totalCount;
          }
          return rowAllocator->finalizeRow(rowSize, result, rowSize);
      }
  };
  
  class CRoxieServerDiskAggregateActivity : public CRoxieServerDiskAggregateBaseActivity
  {
      IHThorCompoundAggregateExtra & aggregateHelper;
  
  public:
      CRoxieServerDiskAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
          unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
            aggregateHelper((IHThorDiskAggregateArg &)basehelper)
      {
      }
  
      const void * gatherMerged()
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
          size32_t finalSize = 0;
          if (useRemote())
          {
              const void * firstRow = remote->nextRow();
              if (!firstRow)
              {
                  rowBuilder.ensureRow();
                  finalSize = aggregateHelper.clearAggregate(rowBuilder);
              }
              else
              {
                   // NOTE need to clone this because going to modify below, could special case 1 row only
                  finalSize = cloneRow(rowBuilder, firstRow, meta);
                  ReleaseRoxieRow(firstRow);
              }
              for (;;)
              {
                  const void * next = remote->nextRow();
                  if (!next)
                      break;
                  finalSize = aggregateHelper.mergeAggregate(rowBuilder, next);
                  ReleaseRoxieRow(next);
              }
          }
          else
          {
              aggregateHelper.clearAggregate(rowBuilder);
              if (helper.canMatchAny() && !eof)
              {
                  for (;;)
                  {
                      const byte *nextRec = reader->nextRow();
                      if (!nextRec)
                          break;
                      aggregateHelper.processRow(rowBuilder, nextRec);
                      reader->finishedRow();
                  }
              }
              finalSize = meta.getRecordSize(rowBuilder.getSelf());
          }
          return rowBuilder.finalizeRowClear(finalSize);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          const void * ret = gatherMerged();
          processed++;
          done = true;
          return ret;
      }
  };
  
  
  class CRoxieServerDiskGroupAggregateActivity : public CRoxieServerDiskAggregateBaseActivity
  {
      IHThorDiskGroupAggregateArg & aggregateHelper;
      RowAggregator resultAggregator;
      bool gathered;
  
  public:
      CRoxieServerDiskGroupAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, IInMemoryIndexManager *_manager, ITranslatorSet *_translators)
          : CRoxieServerDiskAggregateBaseActivity(_ctx, _factory, _probeManager, _remoteId, _numParts, _isLocal, _manager, _translators),
            aggregateHelper((IHThorDiskGroupAggregateArg &)basehelper),
            resultAggregator(aggregateHelper, aggregateHelper),
            gathered(false)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          gathered= false;
          CRoxieServerDiskAggregateBaseActivity::start(parentExtractSize, parentExtract, paused);
          resultAggregator.start(rowAllocator, ctx->queryCodeContext(), activityId);
      }
  
      virtual void reset()
      {
          resultAggregator.reset();
          CRoxieServerDiskAggregateBaseActivity::reset();
      }
  
      void gatherMerged()
      {
          if (useRemote())
          {
              for (;;)
              {
                  const void * next = remote->nextRow();
                  if (!next)
                      break;
                  resultAggregator.mergeElement(next);
                  ReleaseRoxieRow(next);
              }
          }
          else
          {
              if (helper.canMatchAny() && !eof)
              {
                  Owned<IInMemoryFileProcessor> processor = createGroupAggregateRecordProcessor(resultAggregator, aggregateHelper, reader);
                  processor->doQuery(NULL, 0, 0, 0);
              }
          }
          gathered = true;
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done)
              return NULL;
  
          if (!gathered)
              gatherMerged();
  
          Owned<AggregateRowBuilder> next = resultAggregator.nextResult();
          if (next)
          {
              processed++;
              return next->finalizeRowClear();
          }
          done = true;
          return NULL;
      }
  };
  
  
  class CRoxieServerDiskReadActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      RemoteActivityId remoteId;
      bool isLocal;
      bool sorted;
      bool maySkip;
      bool variableFileName;
      Owned<ITranslatorSet> translators;
      Owned<IInMemoryIndexManager> manager;
      Owned<const IResolvedFile> datafile;
      const char *quotes;
      const char *separators;
      const char *terminators;
      const char *escapes;
      size32_t maxCsvRowSize;
  
      CRoxieServerDiskReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, diskStatistics), remoteId(_remoteId)
      {
          isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
          Owned<IHThorDiskReadBaseArg> helper = (IHThorDiskReadBaseArg *) helperFactory();
          sorted = (helper->getFlags() & TDRunsorted) == 0;
          variableFileName = allFilesDynamic || _queryFactory.isDynamic() || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
          maySkip = (helper->getFlags() & (TDRkeyedlimitskips|TDRkeyedlimitcreates|TDRlimitskips|TDRlimitcreates)) != 0;
          quotes = separators = terminators = escapes = NULL;
          if (!variableFileName)
          {
              bool isOpt = (helper->getFlags() & TDRoptional) != 0;
              OwnedRoxieString fileName(helper->getFileName());
              datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
              bool isSimple = (datafile && datafile->getNumParts()==1 && !_queryFactory.queryOptions().disableLocalOptimizations);
              if (isLocal || isSimple)
              {
                  if (datafile)
                  {
                      unsigned channel = isLocal ? queryFactory.queryChannel() : 0;
                      unsigned expectedFormatCrc = helper->getDiskFormatCrc();
                      unsigned projectedFormatCrc = helper->getProjectedFormatCrc();
                      translators.setown(datafile->getTranslators(projectedFormatCrc, helper->queryProjectedDiskRecordSize(), expectedFormatCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), expectedFileMode(), queryFactory.queryQueryName()));
                      manager.setown(datafile->getIndexManager(isOpt, channel, translators->queryActualLayout(0), _graphNode.getPropBool("att[@name=\"preload\"]/@value", false)));
                      const IPropertyTree *options = datafile->queryProperties();
                      if (options)
                      {
                          quotes = options->queryProp("@csvQuote");
                          separators = options->queryProp("@csvSeparate");
                          terminators = options->queryProp("@csvTerminate");
                          escapes = options->queryProp("@csvEscape");
                      }
                  }
                  else
                      manager.setown(createInMemoryIndexManager(helper->queryProjectedDiskRecordSize()->queryRecordAccessor(true), true, nullptr));
              }
          }
          switch (kind)
          {
              case TAKcsvread:
              {
                  maxCsvRowSize = defaultMaxCsvRowSize * 1024 * 1024;
                  IConstWorkUnit *workunit = _queryFactory.queryWorkUnit();
                  if (workunit)
                      maxCsvRowSize = workunit->getDebugValueInt(OPT_MAXCSVROWSIZE, defaultMaxCsvRowSize) * 1024 * 1024;
                  break;
              }
              default:
                  maxCsvRowSize = 0; // unused
          }
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          unsigned numParts = datafile ? datafile->getNumParts() : 0;
          switch (kind)
          {
          case TAKcsvread:
              return new CRoxieServerCsvReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators,
                                                     quotes, separators, terminators, escapes, maxCsvRowSize);
          case TAKxmlread:
          case TAKjsonread:
              return new CRoxieServerXmlReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators);
          case TAKdiskread:
          case TAKspillread:
              return new CRoxieServerDiskReadActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, maySkip, manager, translators);
          case TAKdisknormalize:
              return new CRoxieServerDiskNormalizeActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, sorted, manager, translators);
          case TAKdiskcount:
              return new CRoxieServerDiskCountActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
          case TAKdiskaggregate:
              return new CRoxieServerDiskAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
          case TAKdiskgroupaggregate:
              return new CRoxieServerDiskGroupAggregateActivity(_ctx, this, _probeManager, remoteId, numParts, isLocal, manager, translators);
          }
          throwUnexpected();
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for %s activity", getActivityText(kind));
      }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          if (datafile)
              addXrefFileInfo(reply, datafile);
          else
          {
              // Temporarily resolve the file
              Owned<IHThorDiskReadBaseArg> helper = (IHThorDiskReadBaseArg *) helperFactory();
              if ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) == 0)
              {
                  OwnedRoxieString fileName(helper->getFileName());
                  Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                  if (temp)
                      addXrefFileInfo(reply, temp);
              }
          }
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDiskReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerDiskReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //=================================================================================
  
  class CRoxieServerBaseIndexActivityFactory : public CRoxieServerActivityFactory
  {
  
  public:
      Owned<IKeyArray> keySet;
      Owned<ITranslatorSet> translators;
      RemoteActivityId remoteId;
      bool isSimple;
      bool isLocal;
      bool maySkip;
      bool sorted;
      bool variableFileName;
      unsigned maxSeekLookahead;
      Owned<const IResolvedFile> indexfile;
  
      CRoxieServerBaseIndexActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, indexStatistics), remoteId(_remoteId)
      {
          Owned<IHThorIndexReadBaseArg> indexHelper = (IHThorIndexReadBaseArg *) helperFactory();
          unsigned flags = indexHelper->getFlags();
          sorted = (flags & TIRunordered) == 0;
          isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
          variableFileName = allFilesDynamic || _queryFactory.isDynamic() || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);
          if (!variableFileName)
          {
              bool isOpt = (flags & TIRoptional) != 0;
              OwnedRoxieString indexName(indexHelper->getFileName());
              indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
              if (indexfile)
              {
                  unsigned projectedCrc = indexHelper->getProjectedFormatCrc();
                  unsigned expectedCrc = indexHelper->getDiskFormatCrc();
                  IOutputMetaData *projectedMeta = indexHelper->queryProjectedDiskRecordSize();
                  IOutputMetaData *expectedMeta = indexHelper->queryDiskRecordSize();
                  translators.setown(indexfile->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), FileFormatMode::index, queryFactory.queryQueryName()));
                  keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
              }
          }
          isSimple = isLocal;
          maySkip = (flags & (TIRkeyedlimitskips|TIRlimitskips|TIRlimitcreates|TIRkeyedlimitcreates)) != 0;
  
          if (keySet && keySet->length()==1 && !isLocal && (flags & (TIRlimitskips|TIRlimitcreates|TIRkeyedlimitskips|TIRkeyedlimitcreates))==0)
          {
              IKeyIndexBase *thisBase = keySet->queryKeyPart(0);
              if (thisBase->numParts()==1 && !thisBase->queryPart(0)->isTopLevelKey() && !_queryFactory.queryOptions().disableLocalOptimizations)
                  isSimple = true;
          }
          maxSeekLookahead = _graphNode.getPropInt("hint[@name='maxseeklookahead']/@value", 0);
      }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          if (indexfile)
              addXrefFileInfo(reply, indexfile);
          else
          {
              Owned<IHThorIndexReadBaseArg> indexHelper = (IHThorIndexReadBaseArg *) helperFactory();
              if ((indexHelper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) == 0)
              {
                  OwnedRoxieString indexName(indexHelper->getFileName());
                  Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                  if (temp)
                      addXrefFileInfo(reply, temp);
              }
          }
      }
  
      virtual void setInput(unsigned idx, unsigned source, unsigned sourceidx)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() should not be called for indexread activity");
      }
  };
  
  
  class CRoxieServerIndexActivity : public CRoxieServerActivity, implements IRoxieServerErrorHandler
  {
  protected:
      IHThorIndexReadBaseArg &indexHelper;
      IHThorSteppedSourceExtra *steppedExtra;
      Linked<IKeyArray> keySet;
      Linked<ITranslatorSet> translators;
      CSkippableRemoteResultAdaptor remote;
      CIndexTransformCallback callback;
      bool sorted;
      bool variableFileName;
      bool variableInfoPending;
      bool isOpt;
      bool isLocal;
      unsigned __int64 rowLimit;
      unsigned __int64 keyedLimit;
      unsigned __int64 choosenLimit;
      unsigned accepted;
      unsigned rejected;
      unsigned seekGEOffset;
      Owned<IKeyManager> tlk;
      Owned<const IResolvedFile> varFileInfo;
      const RemoteActivityId &remoteId;
  
      void setVariableFileInfo()
      {
          OwnedRoxieString indexName(indexHelper.getFileName());
          varFileInfo.setown(resolveLFNIndex(indexName, isOpt, defaultPrivilegedUser));
          if (varFileInfo)
          {
              unsigned projectedCrc = indexHelper.getProjectedFormatCrc();
              unsigned expectedCrc = indexHelper.getDiskFormatCrc();
              IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
              IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
              translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), FileFormatMode::index, factory->queryQueryFactory().queryQueryName()));
              keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
          }
          variableInfoPending = false;
      }
  
  public:
      CRoxieServerIndexActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
          bool _sorted, bool _isLocal, bool _maySkip)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            indexHelper((IHThorIndexReadBaseArg &)basehelper),
            keySet(_factory->keySet),
            translators(_factory->translators),
            remote(_ctx, this, _remoteId, meta.queryOriginal(), indexHelper, *this, _sorted, false, _maySkip),
            sorted(_sorted),
            isLocal(_isLocal) ,
            remoteId(_remoteId)
      {
          indexHelper.setCallback(&callback);
          steppedExtra = indexHelper.querySteppingExtra();
          variableFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((indexHelper.getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0);
          variableInfoPending = false;
          isOpt = (indexHelper.getFlags() & TIRoptional) != 0;
          seekGEOffset = 0;
  //        started = false;
          rejected = accepted = 0;
          rowLimit = choosenLimit = keyedLimit = 0;
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          remote.onCreate(_colocalParent);
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          accepted = 0;
          rejected = 0;
          rowLimit = (unsigned __int64) -1;
          keyedLimit = (unsigned __int64 ) -1;
          choosenLimit = I64C(0x7fffffffffffffff);
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          remote.onStart(parentExtractSize, parentExtract);
          variableInfoPending = variableFileName;
      }
  
      void processAllKeys()
      {
          try
          {
              bool noLocal = factory->queryQueryFactory().queryOptions().disableLocalOptimizations || seekGEOffset > 0;
              if (indexHelper.canMatchAny())
              {
                  if (variableInfoPending)
                      setVariableFileInfo();
                  remote.setLimits(rowLimit, keyedLimit, choosenLimit);
                  if (keySet)
                  {
                      // MORE - this recreates the segmonitors per part but not per fileno (which is a little backwards).
                      // With soft layout support may need to recreate per fileno too (i.e. different keys in a superkey have different layout) but never per partno
                      // However order is probably better to iterate fileno's inside partnos 
                      // A superkey that mixes single and multipart or tlk and roroot keys might be hard
                      for (unsigned partNo = 0; partNo < keySet->length(); partNo++)
                      {
                          IKeyIndexBase *thisBase = keySet->queryKeyPart(partNo);
                          if (thisBase)
                          {
                              unsigned fileNo = 0;
                              IKeyIndex *thisKey = thisBase->queryPart(fileNo);
                              if (!thisKey->isTopLevelKey())
                              {
                                  if (keyedLimit != (unsigned __int64) -1)
                                  {
                                      if ((indexHelper.getFlags() & TIRcountkeyedlimit) != 0)
                                      {
                                          Owned<IKeyManager> countKey;
                                          countKey.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this, indexHelper.hasNewSegmentMonitors(), !isBlind()));
                                          countKey->setLayoutTranslator(translators->queryTranslator(fileNo));
                                          createSegmentMonitors(countKey);
                                          unsigned __int64 count = countKey->checkCount(keyedLimit);
                                          if (count > keyedLimit)
                                          {
                                              if (traceLevel > 4)
                                                  DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                                              onLimitExceeded(true);
                                          }
                                      }
                                  }
                              }
                              if (seekGEOffset && !thisKey->isTopLevelKey())
                              {
                                  tlk.setown(createSingleKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, seekGEOffset, this, indexHelper.hasNewSegmentMonitors(), !isBlind()));
                                  tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                              }
                              else
                              {
                                  tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), thisKey, this, indexHelper.hasNewSegmentMonitors(), !isBlind()));
                                  if (!thisKey->isTopLevelKey())
                                      tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                                  else
                                      tlk->setLayoutTranslator(nullptr);
                              }
                              createSegmentMonitors(tlk);
                              if (queryTraceLevel() > 3 || ctx->queryProbeManager())
                              {
                                  StringBuffer out;
                                  tlk->describeFilter(out);
                                  CTXLOG("Using filter %s", out.str());
                                  if (ctx->queryProbeManager())
                                      ctx->queryProbeManager()->setNodeProperty(this, "filter", out.str());
                              }
                              tlk->reset();
                              for (;;) // for each file part
                              {
                                  //block for TransformCallbackAssociation
                                  {
                                      TransformCallbackAssociation associate(callback, tlk);
                                      if (thisKey->isTopLevelKey())
                                      {
                                          if (thisKey->isFullySorted())
                                          {
                                              while (tlk->lookup(false))
                                              {
                                                  unsigned agentPart = (unsigned)extractFpos(tlk);
                                                  if (agentPart)
                                                  {
                                                      accepted++;
                                                      remote.getMem(agentPart, fileNo, 0);  // the cached context is all we need to send
                                                      if (sorted && numChannels>1)
                                                          remote.flush();  // don't combine parts if we need result sorted, except on a 1-way
                                                  }
                                              }
                                          }
                                          else
                                          {
                                              unsigned agentPart = tlk->getPartition();  // Returns 0 if no partition info, or filter cannot be partitioned
                                              remote.getMem(agentPart, fileNo, 0);
                                          }
                                      }
                                      else
                                      {
                                          // Single - part key. We process locally unless smart-stepping is involved, in which case we need to ensure that
                                          // we do the proper two-stage merge.
                                          // Unfortunately processSingleKey will not use the merger we carefully set up before,
                                          // so we disable the local processing in that case
                                          if (noLocal)
                                              remote.getMem(1, fileNo, 0);  // 1 because it's a single part key
                                          else if (processSingleKey(thisKey, translators->queryTranslator(fileNo)))
                                              break;
                                      }
                                  }
                                  if (++fileNo < thisBase->numParts())
                                  {
                                      thisKey = thisBase->queryPart(fileNo);
                                      tlk->setKey(thisKey);
                                      if (!thisKey->isTopLevelKey())
                                          tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                                      else
                                          tlk->setLayoutTranslator(nullptr);
                                      tlk->reset();
                                  }
                                  else
                                      break;
                              }
                              tlk->releaseSegmentMonitors();
                              tlk->setKey(NULL);
                          }
                      }
                  }
              }
              remote.flush();
              remote.senddone();
          }
          catch (IException *E)
          {
              remote.setException(E);
          }
      }
  
      virtual void createSegmentMonitors(IKeyManager *key)
      {
          indexHelper.createSegmentMonitors(key);
          key->finishSegmentMonitors();
      }
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) = 0;
  
      virtual void reset()
      {
          if (accepted)
              noteStatistic(StNumIndexAccepted, accepted);
          if (rejected)
              noteStatistic(StNumIndexRejected, rejected);
          remote.onReset();
          CRoxieServerActivity::reset();
          if (varFileInfo)
          {
              keySet.clear();
              varFileInfo.clear();
          }
          variableInfoPending = false;
      }
  
      virtual void stop()
      {
          remote.onStop();
          CRoxieServerActivity::stop();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          remote.gatherStats(merged);
      }
  };
  
  
  class CRoxieServerIndexReadBaseActivity : public CRoxieServerIndexActivity
  {
  protected:
      IHThorSourceLimitTransformExtra * limitTransformExtra = nullptr;
  public:
      CRoxieServerIndexReadBaseActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
          bool _sorted, bool _isLocal, bool _maySkip)
          : CRoxieServerIndexActivity(_ctx, _factory, _probeManager, _remoteId, _sorted, _isLocal, _maySkip)
      {
      }
  
      virtual void reset()
      {
          remote.onReset();
          CRoxieServerIndexActivity::reset();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          try
          {
              const void *ret = remote.nextRow();
              if (ret)
                  processed++;
              return ret;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
  protected:
      virtual const void * createLimitFailRow(bool isKeyed)
      {
          ensureRowAllocator();
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          assertex(limitTransformExtra);
          size32_t outSize = isKeyed ? limitTransformExtra->transformOnKeyedLimitExceeded(rowBuilder) : limitTransformExtra->transformOnLimitExceeded(rowBuilder);
          if (outSize)
              return rowBuilder.finalizeRowClear(outSize);
          return NULL;
      }
  
  };
  
  class CRoxieServerIndexReadActivity : public CRoxieServerIndexReadBaseActivity, implements IIndexReadActivityInfo 
  {
  protected:
      IHThorIndexReadArg & readHelper;
      ISteppingMeta *rawMeta = nullptr;
      CSteppingMeta steppingMeta;
      unsigned * seekSizes = nullptr;
      bool optimizeSteppedPostFilter;
      ISteppingMeta * projectedMeta = nullptr;
      unsigned maxSeekLookahead;
  
  public:
      CRoxieServerIndexReadActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
          bool _sorted, bool _isLocal, bool _maySkip, unsigned _maxSeekLookahead)
          : CRoxieServerIndexReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _sorted, _isLocal, _maySkip),
            readHelper((IHThorIndexReadArg &)basehelper)
      {
          limitTransformExtra = &readHelper;
          rawMeta = readHelper.queryRawSteppingMeta();
          unsigned flags = indexHelper.getFlags();
          optimizeSteppedPostFilter = (flags & TIRunfilteredtransform) != 0;
          seekSizes = NULL;
          maxSeekLookahead = _maxSeekLookahead;
  
          if (rawMeta)
          {
              const CFieldOffsetSize * fields = rawMeta->queryFields();
              unsigned maxFields = rawMeta->getNumFields();
              seekGEOffset = fields[0].offset;
              seekSizes = new unsigned[maxFields];
              seekSizes[0] = fields[0].size;
              for (unsigned i=1; i < maxFields; i++)
                  seekSizes[i] = seekSizes[i-1] + fields[i].size;
              projectedMeta = readHelper.queryProjectedSteppingMeta();
              ISteppingMeta *useMeta = projectedMeta ? projectedMeta : rawMeta;
              remote.setMergeInfo(useMeta); // also need to consider superfile case where there is a mix of multiway and singleparts.. ?
              bool hasPostFilter = readHelper.transformMayFilter() && optimizeSteppedPostFilter;
              steppingMeta.init(useMeta, hasPostFilter);
          }
      }
  
      ~CRoxieServerIndexReadActivity()
      {
          delete [] seekSizes;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerIndexReadBaseActivity::start(parentExtractSize, parentExtract, paused);
          steppingMeta.setDistributed();
          if (steppedExtra)
              steppingMeta.setExtra(steppedExtra);
          rowLimit = readHelper.getRowLimit();
          keyedLimit = readHelper.getKeyedLimit();
          choosenLimit = readHelper.getChooseNLimit();
          if (!paused)
              processAllKeys();
      }
  
      class LazyLocalKeyReader : implements IMessageUnpackCursor, implements IMessageResult, public CInterface
      {
      public:
          IMPLEMENT_IINTERFACE;
  
          virtual IMessageUnpackCursor *getCursor(roxiemem::IRowManager *rowMgr) const
          {
              Link();
              return const_cast<LazyLocalKeyReader*> (this);
          }
          virtual const void *getMessageHeader(unsigned &length) const
          {
              length = 0;
              return NULL;
          }
          virtual const void *getMessageMetadata(unsigned &length) const
          {
              length = 0;
              return NULL;
          }
          virtual void discard() const
          {
              // nothing to do.
          }
          unsigned keyedCount;
          unsigned matched;
          bool EOFseen;
          Owned<IKeyIndexSet> keySet;
          Owned<IKeyManager> tlk;
          CRoxieServerIndexReadActivity &owner;
          LazyLocalKeyReader(CRoxieServerIndexReadActivity &_owner, IKeyIndex *key, const IDynamicTransform * trans)
              : owner(_owner)
          {
              keyedCount = 0;
              matched = 0;
              EOFseen = false;
              keySet.setown(createKeyIndexSet());
              keySet->addIndex(LINK(key));
              if (owner.seekGEOffset)
                  tlk.setown(createKeyMerger(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet, owner.seekGEOffset, &owner, owner.indexHelper.hasNewSegmentMonitors(), !owner.isBlind()));
              else
                  tlk.setown(createLocalKeyManager(owner.indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keySet->queryPart(0), &owner, owner.indexHelper.hasNewSegmentMonitors(), !owner.isBlind()));
              if (!key->isTopLevelKey())
                  tlk->setLayoutTranslator(trans);
              owner.indexHelper.createSegmentMonitors(tlk);
              tlk->finishSegmentMonitors();
              tlk->reset();
          }
          virtual const void *getNext(int length)
          {
              TransformCallbackAssociation associate(owner.callback, tlk);
              while (tlk->lookup(true))
              {
                  keyedCount++;
                  if (keyedCount > owner.keyedLimit)
                  {
                      owner.onLimitExceeded(true); // Should throw exception
                      throwUnexpected();
                  }
                  size32_t transformedSize;
                  RtlDynamicRowBuilder rowBuilder(owner.rowAllocator);
                  byte const * keyRow = tlk->queryKeyBuffer();
                  if (likely(owner.readHelper.canMatch(keyRow)))
                  {
                      try
                      {
                          transformedSize = owner.readHelper.transform(rowBuilder, keyRow);
                          owner.callback.finishedRow();
                      }
                      catch (IException *E)
                      {
                          throw owner.makeWrappedException(E);
                      }
                      if (transformedSize)
                      {
                          OwnedConstRoxieRow result = rowBuilder.finalizeRowClear(transformedSize);
                          matched++;
                          if (matched > owner.rowLimit)
                          {
                              owner.onLimitExceeded(false); // Should throw exception
                              throwUnexpected();
                          }
                          if (matched > owner.choosenLimit) // MORE - bit of a strange place to check
                          {
                              break;
                          }
                          owner.accepted++;
                          return result.getClear();
                      }
                      else
                          owner.rejected++;
                  }
                  else
                  {
                      owner.callback.finishedRow(); // since filter might have accessed a blob
                      owner.rejected++;
                  }
              }
              EOFseen = true;
              return NULL;
          }
          virtual bool atEOF() const
          {
              return EOFseen;
          }
          virtual bool isSerialized() const
          {
              return false;
          }
      };
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) override
      {
          ensureRowAllocator();
          remote.injectResult(new LazyLocalKeyReader(*this, key, trans));
          return false;
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
          {
              if (indexHelper.getFlags() & (TIRkeyedlimitskips|TIRkeyedlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(true);
              }
              else
                  readHelper.onKeyedLimitExceeded();
          }
          else
          {
              if (indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(false);
              }
              else
                  readHelper.onLimitExceeded();
          }
      }
  
      virtual void serializeSkipInfo(MemoryBuffer &out, unsigned seekLen, const void *rawSeek, unsigned numFields, const void * seek, const SmartStepExtra &stepExtra) const
      {
          out.append((unsigned short) numFields);
          out.append((unsigned short) seekLen);
          out.append((unsigned short) stepExtra.queryFlags());
          IMultipleStepSeekInfo *seeks = stepExtra.queryExtraSeeks();
          if (seeks)
          {
              unsigned lookahead = 40000/seekLen;
              if (maxSeekLookahead && (lookahead > maxSeekLookahead))
                  lookahead  = maxSeekLookahead;
              seeks->ensureFilled(seek, numFields, lookahead);
  
              out.append(lookahead != seeks->ordinality()); // seeksAreEof flag
              unsigned serialized = 1; // rawseek is always serialized...
              unsigned patchLength = out.length();
              out.append(serialized);  // NOTE - we come back and patch with the actual value...
              out.append(seekLen, rawSeek);
              if (seeks->ordinality())
              {
                  const void *lastSeek = rawSeek;
                  byte *nextSeek = NULL;
                  if (projectedMeta)
                      nextSeek = (byte *) alloca(seekLen);
                  for (unsigned i = 0; i < seeks->ordinality(); i++)
                  {
                      if (projectedMeta)
                      {
                          RtlStaticRowBuilder rowBuilder(nextSeek-seekGEOffset, seekGEOffset+seekLen);
                          readHelper.mapOutputToInput(rowBuilder, seeks->querySeek(i), numFields); // NOTE - weird interface to mapOutputToInput means that it STARTS writing at seekGEOffset...
                      }
                      else
                          nextSeek = (byte *) seeks->querySeek(i)+seekGEOffset;
                      int diff = memcmp(nextSeek, lastSeek, seekLen);
                      if (diff > 0)
                      {
                          serialized++;
                          out.append(seekLen, nextSeek);
                          lastSeek = (const byte *) out.reserve(0) - seekLen;
                      }
                  }
                  unsigned length = out.length();
                  out.setWritePos(patchLength);
                  out.append(serialized);
                  out.setWritePos(length);
              }
          }
          else
          {
              out.append(false);
              out.append(1);
              out.append(seekLen, rawSeek);
          }
  
      }
  
      virtual const void * nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          try
          {
              unsigned seeklen = 0;
              const void *rawSeek = NULL;
              if (seek && numFields)
              {
                  seeklen = seekSizes[numFields-1];
                  rawSeek = (const byte *)seek + seekGEOffset;
                  if (projectedMeta)
                  {
                      byte * temp = (byte *) alloca(seeklen);
                      RtlStaticRowBuilder rawBuilder(temp-seekGEOffset, seekGEOffset+seeklen);
                      readHelper.mapOutputToInput(rawBuilder, seek, numFields); // NOTE - weird interface to mapOutputToInput means that it STARTS writing at seekGEOffset...
                      rawSeek = (byte *)temp;
                  }
              }
              const void *ret = remote.nextRowGE(seek, rawSeek, numFields, seeklen, wasCompleteMatch, stepExtra);
              if (ret && wasCompleteMatch) // GH pleas confirm the wasCompleteMatch I just added here is right
                  processed++;
              return ret;
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  
      virtual IInputSteppingMeta * querySteppingMeta()
      {
          if (rawMeta && steppingEnabled && ((indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates|TIRkeyedlimitskips|TIRkeyedlimitcreates)) == 0))
              return &steppingMeta;
          return NULL;
      }
  
      virtual IIndexReadActivityInfo *queryIndexReadActivity() 
      { 
          if (variableInfoPending)
              setVariableFileInfo();
          return this; 
      }
      virtual IKeyArray *getKeySet() const 
      {
          return keySet.getLink();
      }
      virtual const IResolvedFile *getVarFileInfo() const
      {
          return varFileInfo.getLink();
      }
      virtual ITranslatorSet *getTranslators() const override
      {
          return translators.getLink();
      }
      virtual void mergeSegmentMonitors(IIndexReadContext *irc) const
      {
          indexHelper.createSegmentMonitors(irc); // NOTE: they will merge
      }
      virtual IRoxieServerActivity *queryActivity() { return this; }
      virtual const RemoteActivityId& queryRemoteId() const 
      {
          return remoteId;
      }
  };
  
  class CRoxieServerSimpleIndexReadActivity : public CRoxieServerActivity, implements IIndexReadActivityInfo 
  {
      IHThorIndexReadArg & indexHelper;
      IHThorSteppedSourceExtra * steppedExtra;
      bool eof;
      Linked<IKeyArray>keySet;
      Owned<IKeyIndexSet>keyIndexSet;
      Owned<IKeyManager> tlk;
      Linked<ITranslatorSet> translators;
      CIndexTransformCallback callback;
      unsigned __int64 keyedLimit;
      unsigned rowLimit;
      unsigned chooseNLimit;
      unsigned accepted;
      unsigned rejected;
      unsigned keyedCount;
      ISteppingMeta * rawMeta;
      ISteppingMeta * projectedMeta;
      size32_t seekGEOffset;
      unsigned * seekSizes;
      CSteppingMeta steppingMeta;
      Owned<const IResolvedFile> varFileInfo;
      const RemoteActivityId &remoteId;
      bool firstRead;
      bool variableFileName;
      bool variableInfoPending;
      bool isOpt;
      bool isLocal;
      bool optimizeSteppedPostFilter;
      // MORE there may be enough in common between this and CRoxieServerIndexActivity to warrant some refactoring
  
      void initKeySet()
      {
          if (keySet->length() > 1 || rawMeta != NULL)
          {
              if (translators->isTranslatingKeyed())
                  throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Keyed Layout translation is not available when merging key parts or smart-stepping, as it may change record order");
              if (!translators->hasConsistentTranslation())
                  throw MakeStringException(ROXIE_UNIMPLEMENTED_ERROR, "Mixture of layout translation is not available when merging key parts or smart-stepping");
          }
          keyIndexSet.setown(createKeyIndexSet());
          for (unsigned part = 0; part < keySet->length(); part++)
          {
              IKeyIndexBase *kib = keySet->queryKeyPart(part);
              if (kib)
              {
                  for (unsigned subpart = 0; subpart < kib->numParts(); subpart++)
                  {
                      IKeyIndex *k = kib->queryPart(subpart);
                      if (k)
                      {
                          assertex(!k->isTopLevelKey());
                          keyIndexSet->addIndex(LINK(k));
                      }
                  }
              }
          }
      }
  
      void setVariableFileInfo()
      {
          OwnedRoxieString indexName(indexHelper.getFileName());
          varFileInfo.setown(resolveLFNIndex(indexName, isOpt, defaultPrivilegedUser));
          unsigned projectedCrc = indexHelper.getProjectedFormatCrc();
          unsigned expectedCrc = indexHelper.getDiskFormatCrc();
          IOutputMetaData *projectedMeta = indexHelper.queryProjectedDiskRecordSize();
          IOutputMetaData *expectedMeta = indexHelper.queryDiskRecordSize();
          translators.setown(varFileInfo->getTranslators(projectedCrc, projectedMeta, expectedCrc, expectedMeta, getEnableFieldTranslation(), FileFormatMode::index, factory->queryQueryFactory().queryQueryName()));
          keySet.setown(varFileInfo->getKeyArray(isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
          initKeySet();
          variableInfoPending = false;
      }
  
      void onEOF()
      {
          callback.setManager(NULL);
          eof = true;
          tlk.clear();
      }
  
  public:
      CRoxieServerSimpleIndexReadActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, bool _isLocal)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            indexHelper((IHThorIndexReadArg &)basehelper), 
            keySet(_factory->keySet),
            translators(_factory->translators),
            remoteId(_remoteId),
            isLocal(_isLocal)
      {
          rowLimit = 0;
          keyedLimit = 0;
          chooseNLimit = 0;
          indexHelper.setCallback(&callback);
          steppedExtra = indexHelper.querySteppingExtra();
          unsigned flags = indexHelper.getFlags();
          variableFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);
          variableInfoPending = false;
          isOpt = (flags & TIRoptional) != 0;
          optimizeSteppedPostFilter = (flags & TIRunfilteredtransform) != 0;
          firstRead = true;
          accepted = 0;
          rejected = 0;
          keyedCount = 0;
          eof = false;
          rawMeta = indexHelper.queryRawSteppingMeta();
          projectedMeta = indexHelper.queryProjectedSteppingMeta();
          seekGEOffset = 0;
          seekSizes = NULL;
          if (rawMeta)
          {
              // MORE - should check all keys in maxFields list can actually be keyed.
              const CFieldOffsetSize * fields = rawMeta->queryFields();
              unsigned maxFields = rawMeta->getNumFields();
              seekGEOffset = fields[0].offset;
              seekSizes = new unsigned[maxFields];
              seekSizes[0] = fields[0].size;
              for (unsigned i=1; i < maxFields; i++)
                  seekSizes[i] = seekSizes[i-1] + fields[i].size;
              bool hasPostFilter = indexHelper.transformMayFilter() && optimizeSteppedPostFilter;
              if (projectedMeta)
                  steppingMeta.init(projectedMeta, hasPostFilter);
              else
                  steppingMeta.init(rawMeta, hasPostFilter);
          }
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      ~CRoxieServerSimpleIndexReadActivity()
      {
          delete [] seekSizes;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          firstRead = true;
          accepted = 0;
          rejected = 0;
          keyedCount = 0;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (steppedExtra)
              steppingMeta.setExtra(steppedExtra);
          eof = !indexHelper.canMatchAny();
          if (variableFileName)
              variableInfoPending = true;
          else
          {
              variableInfoPending = false;
              if (!keyIndexSet)
                  initKeySet();
          }
      }
  
      virtual IIndexReadActivityInfo *queryIndexReadActivity() 
      { 
          if (variableInfoPending)
              setVariableFileInfo();
          return this; 
      }
      virtual IKeyArray *getKeySet() const 
      {
          return keySet.getLink();
      }
      virtual const IResolvedFile *getVarFileInfo() const
      {
          return varFileInfo.getLink();
      }
      virtual ITranslatorSet *getTranslators() const override
      {
          return translators.getLink();
      }
      virtual void mergeSegmentMonitors(IIndexReadContext *irc) const
      {
          indexHelper.createSegmentMonitors(irc); // NOTE: they will merge
      }
      virtual IRoxieServerActivity *queryActivity() { return this; }
      virtual const RemoteActivityId& queryRemoteId() const 
      {
          return remoteId;
      }
              
      const void *nextRow()
      {
          bool matched = true;
          return nextRowGE(NULL, 0, matched, dummySmartStepExtra);
      }
  
      unsigned __int64 checkCount(unsigned __int64 limit)
      {
          unsigned numParts = keyIndexSet->numParts();
          unsigned __int64 result = 0;
          for (unsigned i = 0; i < numParts; i++)
          {
              Owned<IKeyManager> countTlk = createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(i), this, indexHelper.hasNewSegmentMonitors(), !isBlind());
              countTlk->setLayoutTranslator(translators->queryTranslator(i));
              indexHelper.createSegmentMonitors(countTlk);
              countTlk->finishSegmentMonitors();
              result += countTlk->checkCount(limit-result);
              if (result > limit)
                  break;
          }
          return result;
      }
  
      virtual const void *nextRowGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
          if (firstRead)
          {
              if (variableInfoPending)
                  setVariableFileInfo();
  
              rowLimit = (unsigned) indexHelper.getRowLimit();
              chooseNLimit = (unsigned) indexHelper.getChooseNLimit();
  
              unsigned numParts = keyIndexSet->numParts();
              if (!numParts)
              {
                  onEOF();
                  return NULL;
              }
              if (numParts > 1 || seekGEOffset)
              {
                  tlk.setown(createKeyMerger(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet, seekGEOffset, this, indexHelper.hasNewSegmentMonitors(), !isBlind()));
                  tlk->setLayoutTranslator(translators->queryTranslator(0));
              }
              else
              {
                  tlk.setown(createLocalKeyManager(indexHelper.queryDiskRecordSize()->queryRecordAccessor(true), keyIndexSet->queryPart(0), this, indexHelper.hasNewSegmentMonitors(), !isBlind()));
                  tlk->setLayoutTranslator(translators->queryTranslator(0));
              }
              indexHelper.createSegmentMonitors(tlk);
              tlk->finishSegmentMonitors();
              if (queryTraceLevel() > 3 || ctx->queryProbeManager())
              {
                  StringBuffer out;
                  tlk->describeFilter(out);
                  CTXLOG("Using filter %s", out.str());
                  if (ctx->queryProbeManager())
                      ctx->queryProbeManager()->setNodeProperty(this, "filter", out.str());
              }
              tlk->reset();
              callback.setManager(tlk);
  
              keyedLimit = indexHelper.getKeyedLimit();
              if (keyedLimit != (unsigned __int64) -1)
              {
                  if ((indexHelper.getFlags() & TIRcountkeyedlimit) != 0)
                  {
                      unsigned __int64 count = checkCount(keyedLimit);
                      if (count > keyedLimit)
                      {
                          if ((indexHelper.getFlags() & (TIRkeyedlimitskips|TIRkeyedlimitcreates)) == 0)
                              indexHelper.onKeyedLimitExceeded();
  
                          const void * ret = NULL;
                          if (indexHelper.getFlags() & TIRkeyedlimitcreates)
                              ret = createKeyedLimitOnFailRow();
                          onEOF();
                          return ret;
                      }
                      keyedLimit = (unsigned __int64) -1;
                  }
              }
              firstRead = false;
          }
  
          if (accepted == chooseNLimit)
          {
              onEOF();
              return NULL;
          }
      
          const byte * rawSeek = NULL;
          unsigned seekSize = 0;
          if (seek)
          {
              seekSize = seekSizes[numFields-1];
              rawSeek = (const byte *)seek + seekGEOffset;
              if (projectedMeta)
              {
                  byte *temp = (byte *) alloca(seekSize);
                  RtlStaticRowBuilder rawBuilder(temp-seekGEOffset, seekGEOffset+seekSize);
                  indexHelper.mapOutputToInput(rawBuilder, seek, numFields);// NOTE - weird interface to mapOutputToInput means that it STARTS writing at seekGEOffset...
                  rawSeek = (byte *)temp;
              }
  #ifdef _DEBUG
  //          StringBuffer seekStr;
  //          for (unsigned i = 0; i < seekSize; i++)
  //          {
  //              seekStr.appendf("%02x ", ((unsigned char *) rawSeek)[i]);
  //          }
  //          DBGLOG("nextRowGE can skip offset %d size %d value %s", seekGEOffset, seekSize, seekStr.str());
  #endif
          }
          const byte * originalRawSeek = rawSeek;
          RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
          while (rawSeek ? tlk->lookupSkip(rawSeek, seekGEOffset, seekSize) : tlk->lookup(true))
          {
              checkAbort();
              keyedCount++;
              if (keyedCount > keyedLimit)
              {
                  indexHelper.onKeyedLimitExceeded();
                  break;
              }
  
              byte const * keyRow = tlk->queryKeyBuffer();
              if (likely(indexHelper.canMatch(keyRow)))
              {
  #ifdef _DEBUG
      //          StringBuffer recstr;
      //          unsigned size = (tlk->queryRecordSize()<80)  ? tlk->queryRecordSize() : 80;
      //          for (unsigned i = 0; i < size; i++)
      //          {
      //              recstr.appendf("%02x ", ((unsigned char *) keyRow)[i]);
      //          }
      //          DBGLOG("nextRowGE Got %s", recstr.str());
                  if (originalRawSeek && memcmp(keyRow + seekGEOffset, originalRawSeek, seekSize) < 0)
                      assertex(!"smart seek failure");
  #endif
                  size32_t transformedSize;
                  rowBuilder.ensureRow();
                  try
                  {
                      transformedSize =indexHelper.transform(rowBuilder, keyRow);
                      //if the post filter causes a mismatch, and the stepping condition no longer matches
                      //then return a mismatch record - so the join code can start seeking on the other input.
                      if (transformedSize == 0 && optimizeSteppedPostFilter && stepExtra.returnMismatches())
                      {
                          if (memcmp(keyRow + seekGEOffset, originalRawSeek, seekSize) != 0)
                          {
                              transformedSize = indexHelper.unfilteredTransform(rowBuilder, keyRow);
                              if (transformedSize != 0)
                                  wasCompleteMatch = false;
                          }
                      }
                      callback.finishedRow();
                  }
                  catch (IException *E)
                  {
                      throw makeWrappedException(E);
                  }
                  if (transformedSize)
                  {
                      accepted++;
                      if (accepted > rowLimit)
                      {
                          if ((indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates)) != 0)
                          {
                              throwUnexpected(); // should not have used simple variant if maySkip set...
                          }
                          if (traceLevel > 4)
                              DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                          indexHelper.onLimitExceeded();
                          break;
                      }
                      processed++;
      #ifdef _DEBUG
      //              const byte *ret = (const byte *) out.get();
      //              CommonXmlWriter xmlwrite(XWFnoindent|XWFtrim|XWFopt);
      //              queryOutputMeta()->toXML(ret, xmlwrite);
      //              DBGLOG("ROW: {%p} %s", ret, xmlwrite.str());
      #endif
                      return rowBuilder.finalizeRowClear(transformedSize);
                  }
                  else
                      rejected++;
              }
              else
              {
                  callback.finishedRow(); // since filter might have accessed a blob
                  rejected++;
              }
              rawSeek = NULL;
          }
          onEOF();
          return NULL;
      }
  
      virtual void reset()
      {
          onEOF();
          if (accepted)
              noteStatistic(StNumIndexAccepted, accepted);
          if (rejected)
              noteStatistic(StNumIndexRejected, rejected);
          if (variableFileName)
          {
              varFileInfo.clear();
              translators.clear();
          }
          variableInfoPending = false;
          CRoxieServerActivity::reset();
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
      }
  
      virtual IInputSteppingMeta * querySteppingMeta()
      {
          if (rawMeta && steppingEnabled && ((indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates|TIRkeyedlimitskips|TIRkeyedlimitcreates)) == 0))
              return &steppingMeta;
          return NULL;
      }
  
  protected:
      const void * createKeyedLimitOnFailRow()
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t outSize = indexHelper.transformOnKeyedLimitExceeded(rowBuilder);
          if (outSize)
              return rowBuilder.finalizeRowClear(outSize);
          return NULL;
      }
  };
  
  class CRoxieServerIndexReadActivityFactory : public CRoxieServerBaseIndexActivityFactory
  {
  public:
      CRoxieServerIndexReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerBaseIndexActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (!variableFileName && (keySet==NULL || keySet->length()==0))
              return new CRoxieServerNullActivity(_ctx, this, _probeManager);
          else if (isSimple && !maySkip)
              return new CRoxieServerSimpleIndexReadActivity(_ctx, this, _probeManager, remoteId, isLocal);
          else
              return new CRoxieServerIndexReadActivity(_ctx, this, _probeManager, remoteId, sorted, isLocal, maySkip, maxSeekLookahead);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerIndexReadActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //--------------------------------------------------------------------------------------------------------------------------
  
  class CRoxieServerNullCountActivity : public CRoxieServerActivity
  {
      bool done;
  public:
      CRoxieServerNullCountActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager)
      {
          done = false;
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          done = true;
          size32_t rowSize = meta.getFixedSize();
          void * nullRow = rowAllocator->createRow();
          if (rowSize == 1)
              *(byte *)nullRow = 0;
          else
          {
              assertex(rowSize == sizeof(unsigned __int64));
              *(unsigned __int64 *)nullRow = 0;
          }
          return rowAllocator->finalizeRow(rowSize, nullRow, rowSize);
      }
  };
  
  class CRoxieServerIndexCountActivity : public CRoxieServerIndexActivity
  {
      IHThorIndexCountArg & countHelper;
      bool done;
  
  public:
      CRoxieServerIndexCountActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, bool _isLocal)
          : CRoxieServerIndexActivity(_ctx, _factory, _probeManager, _remoteId, false, _isLocal, false),
            countHelper((IHThorIndexCountArg &)basehelper),
            done(false)
      {
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerIndexActivity::start(parentExtractSize, parentExtract, paused);
          choosenLimit = countHelper.getChooseNLimit();
          rowLimit = countHelper.getRowLimit();
          keyedLimit = countHelper.getKeyedLimit();
          if (!paused)
              processAllKeys();
      }
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) override
      {
          unsigned __int64 count = 0;
          if (countHelper.hasFilter())
          {
              while (tlk->lookup(true))
              {
                  try
                  {
                      count += countHelper.numValid(tlk->queryKeyBuffer());
                      callback.finishedRow();
                  }
                  catch (IException *E)
                  {
                      throw makeWrappedException(E);
                  }
                  accepted++;
                  if (count >= choosenLimit) // MORE - what about limit?
                      break;
              }
          }
          else
              count = tlk->getCount();        //MORE: GH->RKC There should be value in providing a max limit to getCount()
  
          if (count)
          {
              Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult();
              if (count > choosenLimit)
                  count = choosenLimit;
  
              void * recBuffer = rowAllocator->createRow();
              if (meta.getFixedSize() == 1)
                  *(byte *)recBuffer = (byte)count;
              else
              {
                  assertex(meta.getFixedSize() == sizeof(unsigned __int64));
                  *(unsigned __int64 *)recBuffer = count;
              }
              recBuffer = rowAllocator->finalizeRow(meta.getFixedSize(), recBuffer, meta.getFixedSize());
              result->append(recBuffer);
              remote.injectResult(result.getClear());
  
              //GH->RKC for count(,choosen)/exists passing in the previous count would short-circuit this much earlier
              if (count >= choosenLimit)
                  return true;
          }
          return false;
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
          {
              if (indexHelper.getFlags() & (TIRkeyedlimitskips|TIRkeyedlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(true);
              }
              else
              {
                  countHelper.onKeyedLimitExceeded();
              }
          }
          else
          {
              if (indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(false);
              }
              else
              {
                  countHelper.onLimitExceeded();
              }
          }
      }
  
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          throwUnexpected();
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          done = true;
          unsigned __int64 totalCount = 0;
          // Checking keyed limit on server is a bit odd - we only know the counts after postfiltering by the time we are on the server
          // The count needs to behave the same as the output, which will regard a keyed limit as exceeded if the sum of the postfiltered outputs exceeds the keyed limit
          bool hasKeyedLimit =  (keyedLimit != (unsigned __int64) -1);
          bool hasLimit = rowLimit != (unsigned __int64) -1;
  
          try
          {
              for (;;)
              {
                  const void * next = remote.nextRow();
                  if (!next)
                      break;
                  if (meta.getFixedSize() == 1)
                      totalCount += *(byte *)next;
                  else
                      totalCount += *(unsigned __int64 *) next;
                  ReleaseRoxieRow(next);
                  if (totalCount > rowLimit || (totalCount > choosenLimit && !hasLimit && !hasKeyedLimit)) // can't break out early if there is a possibility of later agent throwing limit exception
                      break;
              }
              if (totalCount > rowLimit)
              {
                  unsigned flags = indexHelper.getFlags();
                  if (flags & TIRlimitskips)
                      totalCount = 0;
                  else if (flags & TIRlimitcreates)
                      totalCount = 1;
                  else
                      countHelper.onLimitExceeded();
              }
              else if (totalCount > keyedLimit)
              {
                  unsigned flags = indexHelper.getFlags();
                  if (flags & TIRkeyedlimitskips)
                      totalCount = 0;
                  else if (flags & TIRkeyedlimitcreates)
                      totalCount = 1;
                  else
                      countHelper.onLimitExceeded();
              }
              else if (totalCount > choosenLimit)
                  totalCount = choosenLimit;
          }
          catch (IException *E)
          {
              if (QUERYINTERFACE(E, LimitSkipException))
              {
                  totalCount = 0;
                  unsigned flags = indexHelper.getFlags();
                  if (E->errorCode() == KeyedLimitSkipErrorCode)
                  {
                      if (flags & TIRkeyedlimitcreates)
                          totalCount++;
                  }
                  else
                  {
                      if (flags & TIRlimitcreates)
                          totalCount++;
                  }
                  if (totalCount > choosenLimit)
                      totalCount = choosenLimit; // would have to be weird code (and escape the optimizer...)
                  E->Release();
              }
              else
                  throw ;
          }
  
          processed++;
          void * result = rowAllocator->createRow();
          if (meta.getFixedSize() == 1)
              *(byte *)result = (byte)totalCount;
          else
          {
              assertex(meta.getFixedSize() == sizeof(unsigned __int64));
              *(unsigned __int64 *)result = totalCount;
          }
          return rowAllocator->finalizeRow(meta.getFixedSize(), result, meta.getFixedSize());
      }
  };
  
  class CRoxieServerIndexCountActivityFactory : public CRoxieServerBaseIndexActivityFactory
  {
  public:
      CRoxieServerIndexCountActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerBaseIndexActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (!variableFileName && (keySet==NULL || keySet->length()==0))
              return new CRoxieServerNullCountActivity(_ctx, this, _probeManager);
  //      else if (isSimple)
  //          return new CRoxieServerSimpleIndexCountActivity(this, keySet->queryKeyPart(0)->queryPart(0));
          else
              return new CRoxieServerIndexCountActivity(_ctx, this, _probeManager, remoteId, isLocal);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexCountActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerIndexCountActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //--------------------------------------------------------------------------------------------------------------------------
  
  class CRoxieServerNullIndexAggregateActivity : public CRoxieServerActivity
  {
      IHThorIndexAggregateArg &aggregateHelper;
      bool done;
  public:
      CRoxieServerNullIndexAggregateActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            aggregateHelper((IHThorIndexAggregateArg &)basehelper)
      {
          done = false;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          done = true;
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          size32_t thisSize = aggregateHelper.clearAggregate(rowBuilder);
          return rowBuilder.finalizeRowClear(thisSize);
      }
  };
  
  class CRoxieServerIndexAggregateActivity : public CRoxieServerIndexActivity
  {
      IHThorCompoundAggregateExtra & aggregateHelper;
      bool done;
  
  public:
      CRoxieServerIndexAggregateActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager,
                                        const RemoteActivityId &_remoteId, bool _isLocal)
          : CRoxieServerIndexActivity(_ctx, _factory, _probeManager, _remoteId, false, _isLocal, false),
            aggregateHelper((IHThorIndexAggregateArg &)basehelper),
            done(false)
      {
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          done = false;
          CRoxieServerIndexActivity::start(parentExtractSize, parentExtract, paused);
          if (!paused)
              processAllKeys();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) override
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
          while (tlk->lookup(true))
          {
              if (!rowBuilder.exists())
              {
                  rowBuilder.ensureRow();
                  aggregateHelper.clearAggregate(rowBuilder);
              }
              try
              {
                  aggregateHelper.processRow(rowBuilder, tlk->queryKeyBuffer());
                  callback.finishedRow();
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
              accepted++;
          }
          if (aggregateHelper.processedAnyRows())
          {
              size32_t size = meta.getRecordSize(rowBuilder.getSelf());
              const void * recBuffer = rowBuilder.finalizeRowClear(size);
              Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult();
              result->append(recBuffer);
              remote.injectResult(result.getClear());
          }
          return false;
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          throwUnexpected();
      }
  
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          throwUnexpected();
      }
  
      const void * gatherMerged()
      {
          RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
          const void * firstRow = remote.nextRow();
          size32_t finalSize = 0;
          if (!firstRow)
          {
              rowBuilder.ensureRow();
              finalSize = aggregateHelper.clearAggregate(rowBuilder);
          }
          else
          {
               // NOTE need to clone this because going to modify below, could special case 1 row only
              finalSize = cloneRow(rowBuilder, firstRow, meta);
              ReleaseRoxieRow(firstRow);
          }
          for (;;)
          {
              const void * next = remote.nextRow();
              if (!next)
                  break;
              finalSize = aggregateHelper.mergeAggregate(rowBuilder, next);
              ReleaseRoxieRow(next);
          }
          return rowBuilder.finalizeRowClear(finalSize);
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (done) return NULL;
          const void * ret = gatherMerged();
          processed++;
          done = true;
          return ret;
      }
  };
  
  class CRoxieServerIndexAggregateActivityFactory : public CRoxieServerBaseIndexActivityFactory
  {
  public:
      CRoxieServerIndexAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerBaseIndexActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (!variableFileName && (keySet==NULL || keySet->length()==0))
              return new CRoxieServerNullIndexAggregateActivity(_ctx, this, _probeManager);
  //      else if (isSimple)
  //          return new CRoxieServerSimpleIndexAggregateActivity(this, keySet->queryKeyPart(0)->queryPart(0));
          else
              return new CRoxieServerIndexAggregateActivity(_ctx, this, _probeManager, remoteId, isLocal);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerIndexAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //--------------------------------------------------------------------------------------------------------------------------
  
  class CRoxieServerIndexGroupAggregateActivity : public CRoxieServerIndexActivity, implements IHThorGroupAggregateCallback
  {
      IHThorCompoundGroupAggregateExtra & aggregateHelper;
      RowAggregator singleAggregator;
      RowAggregator resultAggregator;
      unsigned groupSegCount;
      bool gathered;
      bool eof;
  
  public:
      CRoxieServerIndexGroupAggregateActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager,
                                              const RemoteActivityId &_remoteId, bool _isLocal)
          : CRoxieServerIndexActivity(_ctx, _factory, _probeManager, _remoteId, false, _isLocal, false),
            aggregateHelper((IHThorIndexGroupAggregateArg &)basehelper),
            singleAggregator(aggregateHelper, aggregateHelper),
            resultAggregator(aggregateHelper, aggregateHelper),
            gathered(false), eof(true)
      {
          groupSegCount = 0;
      }
  
      IMPLEMENT_IINTERFACE_USING(CRoxieServerIndexActivity)
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          gathered= false;
          CRoxieServerIndexActivity::start(parentExtractSize, parentExtract, paused);
          groupSegCount = 0;
          if (!paused)
              processAllKeys();
          resultAggregator.start(rowAllocator, ctx->queryCodeContext(), activityId);
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void reset()
      {
          resultAggregator.reset();
          CRoxieServerIndexActivity::reset();
      }
  
      virtual void processRow(const void * next)
      {
          singleAggregator.addRow(next);
      }
  
      virtual void createSegmentMonitors(IKeyManager *key)
      {
          ThorActivityKind kind = factory->getKind();
          CRoxieServerIndexActivity::createSegmentMonitors(key);
          if ((kind==TAKindexgroupcount || kind==TAKindexgroupexists)) 
              groupSegCount = aggregateHelper.getGroupingMaxField();
          else
              groupSegCount = 0;
      }
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) override
      {
          Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult();
          singleAggregator.start(rowAllocator, ctx->queryCodeContext(), activityId);
          ThorActivityKind kind = factory->getKind();
          while (tlk->lookup(true))
          {
              try
              {
                  if (groupSegCount && !trans)
                  {
                      AggregateRowBuilder &rowBuilder = singleAggregator.addRow(tlk->queryKeyBuffer());
                      callback.finishedRow();
                      if (kind==TAKindexgroupcount)                   
                      {
                          unsigned __int64 count = tlk->getCurrentRangeCount(groupSegCount);
                          aggregateHelper.processCountGrouping(rowBuilder, count-1);
                      }
                      if (!tlk->nextRange(groupSegCount))
                          break;
                  }
                  else
                  {
                      aggregateHelper.processRow(tlk->queryKeyBuffer(), this);
                      callback.finishedRow();
                  }
              }
              catch (IException *E)
              {
                  throw makeWrappedException(E);
              }
              accepted++;
          }
  
          for (;;)
          {
              Owned<AggregateRowBuilder> next = singleAggregator.nextResult();
              if (!next)
                  break;
              result->append(next->finalizeRowClear());
          }
          remote.injectResult(result.getClear());
          singleAggregator.reset();
          return false;
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          throwUnexpected();
      }
  
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          throwUnexpected();
      }
  
      void gatherMerged()
      {
          gathered = true;
          for (;;)
          {
              const void * next = remote.nextRow();
              if (!next)
                  break;
              resultAggregator.mergeElement(next);
              ReleaseRoxieRow(next);
          }
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if (eof)
              return NULL;
  
          if (!gathered)
              gatherMerged();
  
          Owned<AggregateRowBuilder> next = resultAggregator.nextResult();
          if (next)
          {
              processed++;
              return next->finalizeRowClear();
          }
          eof = true;
          return NULL;
      }
  };
  
  class CRoxieServerIndexGroupAggregateActivityFactory : public CRoxieServerBaseIndexActivityFactory
  {
  public:
      CRoxieServerIndexGroupAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerBaseIndexActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (!variableFileName && (keySet==NULL || keySet->length()==0))
              return new CRoxieServerNullActivity(_ctx, this, _probeManager);
  //      else if (isSimple)
  //          return new CRoxieServerSimpleIndexGroupAggregateActivity(this, keySet->queryKeyPart(0)->queryPart(0));
          else
              return new CRoxieServerIndexGroupAggregateActivity(_ctx, this, _probeManager, remoteId, isLocal);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexGroupAggregateActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerIndexGroupAggregateActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //--------------------------------------------------------------------------------------------------------------------------
  
  class CRoxieServerIndexNormalizeActivity : public CRoxieServerIndexReadBaseActivity
  {
      IHThorIndexNormalizeArg & readHelper;
  
  public:
      CRoxieServerIndexNormalizeActivity(IRoxieAgentContext *_ctx, const CRoxieServerBaseIndexActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId,
                                         bool _sorted, bool _isLocal, bool _maySkip)
          : CRoxieServerIndexReadBaseActivity(_ctx, _factory, _probeManager, _remoteId, _sorted, _isLocal, _maySkip),
            readHelper((IHThorIndexNormalizeArg &)basehelper)
      {
          limitTransformExtra = &readHelper;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerIndexReadBaseActivity::start(parentExtractSize, parentExtract, paused);
          rowLimit = readHelper.getRowLimit();
          keyedLimit = readHelper.getKeyedLimit();
          choosenLimit = readHelper.getChooseNLimit();
          if (!paused)
              processAllKeys();
      }
  
      virtual bool processSingleKey(IKeyIndex *key, const IDynamicTransform * trans) override
      {
          unsigned keyedCount = 0;
          RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
          while (tlk->lookup(true))
          {
              keyedCount++;
              if (keyedCount > keyedLimit)
              {
                  if (traceLevel > 4)
                      DBGLOG("activityid = %d  line = %d", activityId, __LINE__);
                  onLimitExceeded(true); 
                  break;
              }
              size32_t transformedSize;
      
              if (readHelper.first(tlk->queryKeyBuffer()))
              {
                  Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult();
                  do
                  {
                      if (accepted>=choosenLimit)
                          break;
                      rowBuilder.ensureRow();
                      try
                      {
                          transformedSize = readHelper.transform(rowBuilder);
                      }
                      catch (IException *E)
                      {
                          throw makeWrappedException(E);
                      }
  
                      if (transformedSize)
                      {
                          // MORE - would be a good idea to stop these asap if rowlimit exceeded
                          result->append(rowBuilder.finalizeRowClear(transformedSize));
                          accepted++;
                      }
                      else
                          rejected++;
                  } while (readHelper.next());
                  remote.injectResult(result.getClear());
                  callback.finishedRow();
                  if (accepted>=choosenLimit)
                      return true;
              }
          }
          return false;
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
          {
              if (indexHelper.getFlags() & (TIRkeyedlimitskips|TIRkeyedlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(true);
              }
              else
                  readHelper.onKeyedLimitExceeded();
          }
          else
          {
              if (indexHelper.getFlags() & (TIRlimitskips|TIRlimitcreates))
              {
                  if (ctx->queryDebugContext())
                      ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
                  throw makeLimitSkipException(false);
              }
              else
                  readHelper.onLimitExceeded();
          }
      }
  
  };
  
  class CRoxieServerIndexNormalizeActivityFactory : public CRoxieServerBaseIndexActivityFactory
  {
  public:
      CRoxieServerIndexNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerBaseIndexActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (!variableFileName && (keySet==NULL || keySet->length()==0))
              return new CRoxieServerNullActivity(_ctx, this, _probeManager);
          else
              return new CRoxieServerIndexNormalizeActivity(_ctx, this, _probeManager, remoteId, sorted, isLocal, maySkip);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerIndexNormalizeActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerIndexNormalizeActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  //=================================================================================
  
  class CRoxieServerFetchActivity : public CRoxieServerActivity, implements IRecordPullerCallback, implements IRoxieServerErrorHandler
  {
      IHThorFetchBaseArg &helper;
      Linked<IFilePartMap> map;
      CRemoteResultAdaptor remote;
      RecordPullerThread puller;
      bool needsRHS;
      bool variableFileName;
      bool isOpt;
      Owned<const IResolvedFile> varFileInfo;
      Owned<IEngineRowAllocator> extractAllocator;
  
  public:
      CRoxieServerFetchActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, IFilePartMap *_map)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorFetchBaseArg &)basehelper), map(_map), remote(_ctx, this, _remoteId, meta.queryOriginal(), helper, *this, true, true), puller(false)
      {
          needsRHS = helper.transformNeedsRhs();
          if (needsRHS)
              extractAllocator.setown(createRowAllocator(helper.queryExtractedSize()));
  
          variableFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
          isOpt = (helper.getFetchFlags() & FFdatafileoptional) != 0;
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          remote.onCreate(_colocalParent);
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (idx)
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
          puller.setInput(this, _sourceIdx, _in);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          puller.connectInputStreams(ctx, consumerOrdered);
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return puller.queryInput();
          else
              return NULL;
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          remote.gatherStats(merged);
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          remote.onStart(parentExtractSize, parentExtract);
          remote.setLimits(helper.getRowLimit(), (unsigned __int64) -1, I64C(0x7FFFFFFFFFFFFFFF));
          if (variableFileName)
          {
              OwnedRoxieString fname(helper.getFileName());
              varFileInfo.setown(resolveLFNFlat(fname, isOpt, defaultPrivilegedUser));
              if (varFileInfo)
                  map.setown(varFileInfo->getFileMap());
          }
          puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().fetchPreload, false, ctx);
      }
  
      virtual void stop()
      {
          // Called from remote, so no need to call back to it....
          puller.stop();
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          processed = remote.processed;
          remote.processed = 0;
          puller.reset();
          if (variableFileName)
          {
              varFileInfo.clear();
              map.clear();
          }
          CRoxieServerActivity::reset();
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL : &remote;
      }
  
      virtual void processRow(const void *row)
      {
          // called from puller thread
          offset_t rp = helper.extractPosition(row);
          unsigned partNo;
          if (isLocalFpos(rp))
              partNo = getLocalFposPart(rp) + 1;
          else
              partNo = map->mapOffset(rp);
          if (needsRHS)
          {
              RtlDynamicRowBuilder rb(extractAllocator, true);
              unsigned rhsSize = helper.extractJoinFields(rb, row);
              char * block = (char *) remote.getMem(partNo, 0, sizeof(rp) + sizeof(rhsSize) + rhsSize); // MORE - superfiles
              *(offset_t *) block = rp;
              block += sizeof(rp);
              *(unsigned *) block = rhsSize;
              block += sizeof(rhsSize);
              memcpy(block, rb.row(), rhsSize);
          }
          else
              *(offset_t *) remote.getMem(partNo, 0, sizeof(rp)) = rp; // MORE - superfiles
          ReleaseRoxieRow(row);
      }
  
      void processEOG()
      {
  #ifdef FETCH_PRESERVES_GROUPING
          UNIMPLEMENTED;
  #endif
          // else discard is correct
      }
  
      void processGroup(const ConstPointerArray &)
      {
          throwUnexpected();
      }
  
      void processDone()
      {
          // called from puller thread
          remote.flush();
          remote.senddone();
      }
  
      virtual bool fireException(IException *e)
      {
          return remote.fireException(e);
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
              throwUnexpected();
          helper.onLimitExceeded();
      }
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          UNIMPLEMENTED;
      }
  
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  };
  
  class CRoxieServerFetchActivityFactory : public CRoxieServerActivityFactory
  {
      RemoteActivityId remoteId;
      Owned<IFilePartMap> map;
      bool variableFileName;
      Owned<const IResolvedFile> datafile;
  public:
      CRoxieServerFetchActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, diskStatistics), remoteId(_remoteId)
      {
          Owned<IHThorFetchBaseArg> helper = (IHThorFetchBaseArg *) helperFactory();
          variableFileName = allFilesDynamic || _queryFactory.isDynamic() || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
          if (!variableFileName)
          {
              OwnedRoxieString fname(helper->getFileName());
              datafile.setown(_queryFactory.queryPackage().lookupFileName(fname,
                                                                          (helper->getFetchFlags() & FFdatafileoptional) != 0,
                                                                          true, true,
                                                                          _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
              if (datafile)
                  map.setown(datafile->getFileMap());
          }
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerFetchActivity(_ctx, this, _probeManager, remoteId, map);
      }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          if (datafile)
              addXrefFileInfo(reply, datafile);
          else
          {
              // Temporarily resolve the file
              Owned<IHThorFetchBaseArg> helper = (IHThorFetchBaseArg *) helperFactory();
              if ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) == 0)
              {
                  OwnedRoxieString fileName(helper->getFileName());
                  Owned<const IResolvedFile> temp = queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned());
                  if (temp)
                      addXrefFileInfo(reply, temp);
              }
          }
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerFetchActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId)
  {
      return new CRoxieServerFetchActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId);
  }
  
  // MORE - is there any point keeping this now?
  
  class CRoxieServerDummyActivityFactory : public CRoxieServerActivityFactory  // not a real activity - just used to properly link files
  {
  public:
      Owned<const IResolvedFile> indexfile;
      Owned<const IResolvedFile> datafile;
      StringAttr fileName;
      StringAttr indexName;
      Owned<IKeyArray> keySet;
      Owned<IFileIOArray> files;
  
      CRoxieServerDummyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool isLoadDataOnly)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, allStatistics)
      {
          try  // does not want any missing file errors to be fatal, or throw traps - just log it
          {
              if (_graphNode.getPropBool("att[@name='_isSpill']/@value", false) || _graphNode.getPropBool("att[@name='_isSpillGlobal']/@value", false))
                  return;  // ignore 'spills'
              bool isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
              ThorActivityKind kind = getActivityKind(_graphNode);
              if (kind != TAKdiskwrite && kind != TAKspillwrite && kind != TAKindexwrite && kind != TAKpiperead && kind != TAKpipewrite)
              {
                  fileName.set(queryNodeFileName(_graphNode, kind));
                  indexName.set(queryNodeIndexName(_graphNode, kind));
                  if (indexName && !allFilesDynamic && !queryFactory.isDynamic())
                  {
                      bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
                      indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                      if (indexfile)
                          keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
                  }
                  if (fileName && !allFilesDynamic && !queryFactory.isDynamic())
                  {
                      bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
                      datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                      if (datafile)
                      {
                          if (isLocal)
                              files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
                      }
                  }
              }
          }
          catch(IException *E)
          {
              StringBuffer errors;
              E->errorMessage(errors);
              OERRLOG("%s File error = %s", (isLoadDataOnly) ? "LOADDATAONLY" : "SUSPENDED QUERY", errors.str());
              E->Release();
          }
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const { throw MakeStringException(ROXIE_INTERNAL_ERROR, "%s query %s is suspended and cannot be executed - error occurred at %s(%d)", (queryFactory.isQueryLibrary()) ? "Library" : " ", queryFactory.queryQueryName(), sanitizeSourceFile(__FILE__), __LINE__); }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          if (!allFilesDynamic)
          {
              if (datafile)
                  addXrefFileInfo(reply, datafile);
              if (indexfile)
                  addXrefFileInfo(reply, indexfile);
          }
          else
          {
              Owned<const IResolvedFile> temp;
              if (fileName.length())
              {
                  temp.setown(queryFactory.queryPackage().lookupFileName(fileName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                  if (temp)
                      addXrefFileInfo(reply, temp);
              }
              if (indexName.length())
              {
                  temp.setown(queryFactory.queryPackage().lookupFileName(indexName, true, true, false, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
                  if (temp)
                      addXrefFileInfo(reply, temp);
              }
          }
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerDummyActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool isLoadDataOnly)
  {
      return new CRoxieServerDummyActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, isLoadDataOnly);
  }
  
  //=====================================================================================================
  
  // Keyed joins... 
  //
  // Input records are pulled by a puller thread, which checks each LHS record to determine which (if any) channels it 
  // may have RHS matches on, and sends the relevant fields to the relevant agents.
  // A separate thread (the caller's thread) is waiting on agent replies, and once it has all replies for a given LHS record or group of records, calls
  // the transform and returns rows that are created.
  // For a full-keyed join, there is a third thread that is pulling replies from index part and passing them to fetch part (check this is true)
  //  
  //=====================================================================================================
  
  class CJoinGroup;
  
  interface IJoinProcessor
  {
      virtual void processEOG() = 0;
      virtual CJoinGroup *createJoinGroup(const void *row) = 0;
      virtual void noteEndReceived(CJoinGroup *jg, unsigned candidateCount) = 0;
      virtual bool fireException(IException *E) = 0;
      virtual void processCompletedGroups() = 0;
  };
  
  //------------------------------------------------------------------------------------------------------
  // Class CJoinGroup has a record per LHS row, plus (if preserving grouping) a 'head of group' record
  // It gathers all the corresponding RHS rows, keeping track of how may agent transactions are pending in endMarkersPending
  // If preserving groups, the 'head of group' record keeps track of how many LHS records in the group are still incomplete.
  // CJoinGroup records are allocated out of the Roxie row memory manager by overloading operator new, so that they are included in the 
  // per-query limits etc (Note that the pointer array block is not though).
  // Because of that, the exact size is significant - especially whether fit just under or just over a chunking threshold...
  //
  // There are two phases to the life of a JoinGroup - it is created by the puller thread that is also firing off agent requests
  // notePending will be called once for every agent request. Puller thread calls noteEndReceived(0) once when done - this corresponds to the
  // initial count when created.
  // Agent replies and are noted by the consumer thread calling addRightMatch() and noteEndReceived(n).
  // Once endMarkersPending reaches 0, JoinGroup is complete. Last thread to call noteEndReceived will process the rows and destroy the group.
  // There is no need for a critsec because although multiple threads will access at different times, only the consumer thread will
  // access any modifiable member variables while endMarkersPending != 0 (i.e. complete() is false). Once complete returns true there is a single
  // remaining reference and the JoinGroup will be processed and destroyed.
  //   
  //------------------------------------------------------------------------------------------------------
  
  class CJoinGroup : public CInterface
  {
  protected:
      const void *left;                   // LHS row
      PointerArrayOf<KeyedJoinHeader> rows;           // matching RHS rows
      std::atomic<unsigned> endMarkersPending; // How many agent responses still waiting for
      CJoinGroup *groupStart;     // Head of group, or NULL if not grouping
      unsigned lastPartNo;
      unsigned pos;
      unsigned candidates;        // Number of RHS keyed candidates - note this may not be the same as rows.ordinality()
  
  public:
  
      void *operator new(size_t size, IRowManager *a, unsigned activityId)
      {
          return a->allocate(size, activityId);
      }
      void operator delete(void *ptr, IRowManager *a, unsigned activityId)
      {
          ReleaseRoxieRow(ptr);
      }
      void operator delete(void *ptr)
      { 
          ReleaseRoxieRow(ptr);
      }
  
  public:
      CJoinGroup(const void *_left, CJoinGroup *_groupStart)
      {
  #ifdef TRACE_JOINGROUPS
          DBGLOG("Creating joinGroup %p, groupstart %p", this, _groupStart);
  #endif
          candidates = 0;
          lastPartNo = 0;
          pos = 0;
          left = _left;
          groupStart = _groupStart;
          if (_groupStart)
          {
              ++_groupStart->endMarkersPending;
          }
          endMarkersPending = 1;
      }
  
      ~CJoinGroup()
      {
  #ifdef TRACE_JOINGROUPS
          DBGLOG("Destroying joinGroup %p", this);
  #endif
          if (left)
          {
              ReleaseRoxieRow(left);
              roxiemem::ReleaseRoxieRowArray(rows.ordinality(), (const void * *)rows.getArray());
              rows.kill();
          }
      }
  
      inline bool isHeadRecord() const
      {
          return left==NULL;
      }
  
      inline bool complete() const
      {
          return endMarkersPending == 0;
      }
  
  #ifdef TRACE_JOINGROUPS
      inline void notePending(unsigned lineNo)
  #else
      inline void notePending()
  #endif
      {
          assert(!complete());
          ++endMarkersPending;
  #ifdef TRACE_JOINGROUPS
          DBGLOG("CJoinGroup::notePending %p from %d, count became %d group count %d", this, lineNo, endMarkersPending.load(), groupStart ? groupStart->endMarkersPending.load() : 0);
  #endif
      }
  
      inline bool inGroup(CJoinGroup *leader) const
      {
          return groupStart==leader;
      }
  
      inline const KeyedJoinHeader *queryRow(unsigned idx) const
      {
          // Single threaded by now
          assert(complete());
          return rows.item(idx);
      }
  
  #ifdef TRACE_JOINGROUPS
      bool noteEndReceived(unsigned candidateCount, unsigned lineNo)
  #else
      bool noteEndReceived(unsigned candidateCount)
  #endif
      {
          assert(!complete());
          if (candidateCount)
          {
              candidates += candidateCount;
          }
  #ifdef TRACE_JOINGROUPS
          DBGLOG("CJoinGroup::noteEndReceived %p from %d, candidates %d + %d, my count was %d, group count was %d", this, lineNo, candidates, candidateCount, endMarkersPending.load(), groupStart ? groupStart->endMarkersPending.load() : 0);
  #endif
          // NOTE - as soon as endMarkersPending and groupStart->endMarkersPending are decremented to zero this object may get released asynchronously by other threads
          // There must therefore be nothing in this method after them that acceses member variables. Think of it as a delete this...
          // In particular, we can't safely reference groupStart after the dec_and_test of endMarkersPending, hence copy local first 
          CJoinGroup *localGroupStart = groupStart;
          if (--endMarkersPending == 0)
          {
              if (localGroupStart)
                  return --localGroupStart->endMarkersPending == 0;
              else
                  return true;
          }
          else
              return false;
      }
  
      inline const void *queryLeft() const
      {
          return left;
      }
  
      void addRightMatch(KeyedJoinHeader *right)
      {
          assert(!complete());
          unsigned short partNo = right->partNo;
          if (partNo != lastPartNo)
          {
              // MORE  - should we binchop? If we did we would need to be careful to find LAST match
              if (partNo > lastPartNo)
                  pos = rows.length();
              while (pos>0)
              {
                  if (rows.item(pos-1)->partNo <= partNo)
                      break;
                  pos--;
              }
              lastPartNo = partNo;
          }
          rows.add(right, pos);
          pos++;
      }
  
      inline unsigned rowsSeen() const
      {
          assert(complete());
          return rows.length();
      }
  
      inline unsigned candidateCount() const
      {
          assert(complete());
          return candidates;
      }
  };
  
  #ifdef TRACE_JOINGROUPS
  #define notePending() notePending(__LINE__)
  #define noteEndReceived(a) noteEndReceived(a, __LINE__)
  #endif
  
  class KeyedJoinRemoteAdaptor : public CRemoteResultAdaptor // MORE - not sure it should be derived from this - makes processed all wrong, for example
  {
  private:
      SafeQueueOf<const void, true> ready;
  
  public:
      IHThorKeyedJoinArg &helper;
  
      unsigned joinProcessed;
      bool isFullKey;
      bool eof;
      bool isSimple;
      bool allPulled;
      ActivityTimeAccumulator activityStats;
      unsigned activityId;
      RecordPullerThread &puller;
      SafeQueueOf<const void, true> injected; // Used in isSimple mode
      Owned<IEngineRowAllocator> ccdRecordAllocator;
      IJoinProcessor &processor;
  
      KeyedJoinRemoteAdaptor(IRoxieAgentContext *_ctx, IRoxieServerErrorHandler *_errorHandler, const RemoteActivityId &_remoteId, IHThorKeyedJoinArg &_helper,
                             IRoxieServerActivity &_activity, bool _isFullKey, bool _isSimple,
                             RecordPullerThread &_puller, IJoinProcessor &_processor)
          : CRemoteResultAdaptor(_ctx, _errorHandler, _remoteId, 0, _helper, _activity, true, true),
            helper(_helper),
            isFullKey(_isFullKey),
            isSimple(_isSimple),
            activityId(_activity.queryId()),
            puller(_puller),
            processor(_processor)
      {
          joinProcessed = 0;
          allPulled = false;
          eof = false;
      }
  
      virtual void onCreate(IHThorArg *_colocalArg)
      {
          CRemoteResultAdaptor::onCreate(_colocalArg);
          ccdRecordAllocator.setown(activity.createRowAllocator(helper.queryJoinFieldsRecordSize()));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          joinProcessed = 0;
          activityStats.totalCycles = 0;
          allPulled = false;
          assertex(ready.ordinality()==0);
          CRemoteResultAdaptor::start(parentExtractSize, parentExtract, paused);
      }
  
      virtual void reset()
      {
          CRemoteResultAdaptor::reset();
          while (ready.ordinality())
          {
              const void *goer = ready.dequeue();
              if (goer)
                  ReleaseRoxieRow(goer);
          }
          while (injected.ordinality())
          {
              const void *goer = injected.dequeue();
              if (goer)
                  ReleaseRoxieRow(goer);
          }
      }
  
      inline void addResult(const void *row)
      {
          ready.enqueue(row);
      }
  
      virtual unsigned __int64 queryTotalCycles() const
      {
          return activityStats.totalCycles;
      }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          for (;;)
          {
              if (eof)
                  return NULL;
              processAgentResults();
              if (ready.ordinality())
              {
                  const void *result = ready.dequeue();
                  if (result)
                      joinProcessed++;
                  return result;
              }
              else
                  eof = true;
          }
      }
  
      void gatherStats(CRuntimeStatisticCollection & merged) const
      {
          CRemoteResultAdaptor::gatherStats(merged);
          if (ccdRecordAllocator)
              ccdRecordAllocator->gatherStats(merged);
      }
  
  private:
      void processAgentResults()
      {
          while (!ready.ordinality())
          {
              KeyedJoinHeader *fetchedData;
              if (isSimple)
              {
                  while (!allPulled && !injected.ordinality())
                  {
                      if (!puller.pullRecords(1))
                      {
                          puller.done();
                          allPulled = true;
                      }
                  }
                  fetchedData = (KeyedJoinHeader *) injected.dequeue();
              }
              else
                  fetchedData = (KeyedJoinHeader *) CRemoteResultAdaptor::nextRow();
              if (fetchedData)
              {
                  CJoinGroup *thisGroup = fetchedData->thisGroup;
                  if (fetchedData->partNo == (unsigned short) -1)
                  {
  #ifdef TRACE_JOINGROUPS
                      CTXLOG("Got end for group %p", thisGroup);
  #endif
                      unsigned candidateCount = (unsigned) fetchedData->fpos;
                      ReleaseRoxieRow(fetchedData);
                      processor.noteEndReceived(thisGroup, candidateCount); // note - this can throw exception. So release fetchdata before calling
                  }
                  else
                  {
  #ifdef TRACE_JOINGROUPS
                      CTXLOG("Reading another %d bytes for group %p data", ccdRecordSize, thisGroup);
  #endif
                      thisGroup->addRightMatch(fetchedData);
                      if (isFullKey)
                      {
  #ifdef TRACE_JOINGROUPS
                          CTXLOG("Calling noteEndReceived for record returned from FETCH of full keyed join");
  #endif
                          processor.noteEndReceived(thisGroup, 0); // note - this can throw exception. So release fetchdata before calling
                      }
                  }
              }
              else
                  break;
          }
      }
  };
  
  class CRoxieServerFullKeyedJoinHead: public CRoxieServerActivity, implements IRecordPullerCallback, implements IRoxieServerErrorHandler
  {
      IHThorKeyedJoinArg &helper;
      Owned<IKeyManager> tlk;
      Linked<IKeyArray> keySet;
      Linked<ITranslatorSet> translators;
      CRemoteResultAdaptor remote;
      RecordPullerThread puller;
      IOutputMetaData *indexReadMeta;
      IJoinProcessor *joinHandler;
      bool variableIndexFileName;
      bool indexReadInputRecordVariable;
      bool isLocal;
      Owned<IEngineRowAllocator> indexReadAllocator;
      Owned<const IResolvedFile> varFileInfo;
      IFinalRoxieInput *indexReadInput;
      unsigned indexReadIdx = 0;
      IIndexReadActivityInfo *rootIndex;
  
  public:
      CRoxieServerFullKeyedJoinHead(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, IKeyArray * _keySet, ITranslatorSet *_translators, IOutputMetaData *_indexReadMeta, IJoinProcessor *_joinHandler, bool _isLocal)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            helper((IHThorKeyedJoinArg &)basehelper), 
            tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this, helper.hasNewSegmentMonitors(), !isBlind())),
            keySet(_keySet),
            translators(_translators),
            remote(_ctx, this, _remoteId, 0, helper, *this, true, true),
            puller(false),
            indexReadMeta(_indexReadMeta),
            joinHandler(_joinHandler),
            isLocal(_isLocal)
      {
          variableIndexFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
          indexReadInputRecordVariable = indexReadMeta->isVariableSize();
          indexReadInput = NULL;
          rootIndex = NULL;
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          remote.onCreate(_colocalParent);
          indexReadAllocator.setown(createRowAllocator(indexReadMeta));
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (!idx)
              puller.setInput(this, _sourceIdx, _in);
          else if (idx==1)
          {
              indexReadInput = _in;
              indexReadIdx = _sourceIdx;
          }
          else
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          puller.connectInputStreams(ctx, consumerOrdered);
          //No rows are read from indexReadInput, so no need to extract the streams
          return CRoxieServerActivity::getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          switch (idx)
          {
          case 0:
              return puller.queryInput();
          case 1:
              return indexReadInput;
          default:
              return NULL;
          }
      }
  
      virtual void serializeExtra(MemoryBuffer &out)
      {
          if (helper.getJoinFlags() & JFindexfromactivity)
          {
              assertex(rootIndex);
              const RemoteActivityId& indexId = rootIndex->queryRemoteId();
              indexId.serialize(out);
              // could mess about reserving space for length then patching it again, to avoid copy, but probably not worth it
              MemoryBuffer tmp;
              rootIndex->queryActivity()->serializeCreateStartContext(tmp);
              if (rootIndex->queryActivity()->queryVarFileInfo())
              {
                  rootIndex->queryActivity()->queryVarFileInfo()->queryTimeStamp().serialize(tmp);
                  tmp.append(rootIndex->queryActivity()->queryVarFileInfo()->queryCheckSum());
              }
              unsigned ctxlen = tmp.length();
              out.append(ctxlen).append(tmp);
          }
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (indexReadInput)
          {
              indexReadInput->start(parentExtractSize, parentExtract, true); // paused=true because we don't want to actually run the index read
              rootIndex = indexReadInput->queryIndexReadActivity();
              if (!rootIndex)
                  throw MakeStringException(ROXIE_INTERNAL_ERROR,"Index in keyed join %d could not be resolved", queryId());
          }
          remote.onStart(parentExtractSize, parentExtract);
          remote.setLimits(helper.getRowLimit(), (unsigned __int64) -1, I64C(0x7FFFFFFFFFFFFFFF));
          if (rootIndex)
          {
              varFileInfo.setown(rootIndex->getVarFileInfo());
              translators.setown(rootIndex->getTranslators());
              keySet.setown(rootIndex->getKeySet());
          }
          else if (variableIndexFileName)
          {
              OwnedRoxieString indexFileName(helper.getIndexFileName());
              varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0, factory->isActivityCodeSigned()));
              if (varFileInfo)
              {
                  unsigned expectedCrc = helper.getIndexFormatCrc();
                  unsigned projectedCrc = helper.getProjectedIndexFormatCrc();
                  translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), getEnableFieldTranslation(), FileFormatMode::index, factory->queryQueryFactory().queryQueryName()));
                  keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
              }
          }
          puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().fullKeyedJoinPreload, false, ctx);
      }
  
      virtual void stop()
      {
          puller.stop();
          CRoxieServerActivity::stop();
      }
  
      virtual void reset()
      {
          CRoxieServerActivity::reset();
          puller.reset();
          if (varFileInfo)
          {
              keySet.clear();
              varFileInfo.clear();
          }
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL : &remote;
      }
  
      virtual void processRow(const void *row)
      {
          // MORE - this code seems to be pretty much duplicated below in half-keyed....
  
          // called from front puller thread
          // buffer up an IndexRead request
          if (keySet && helper.leftCanMatch(row))
          {
              RtlDynamicRowBuilder extractedBuilder(indexReadAllocator);
              unsigned indexReadSize = helper.extractIndexReadFields(extractedBuilder, row);
              OwnedConstRoxieRow extracted;
              if (indexReadSize)
                  extracted.setown(extractedBuilder.finalizeRowClear(indexReadSize));
  
              CJoinGroup *jg = joinHandler->createJoinGroup(row);
              for (unsigned partNo = 0; partNo < keySet->length(); partNo++)
              {
                  IKeyIndexBase *thisBase = keySet->queryKeyPart(partNo);
                  if (thisBase)
                  {
                      unsigned fileNo = 0;
                      IKeyIndex *thisKey = thisBase->queryPart(fileNo);
                      try
                      {
                          tlk->setKey(thisKey);
                          if (!thisKey->isTopLevelKey())
                              tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                          else
                              tlk->setLayoutTranslator(nullptr);
                          helper.createSegmentMonitors(tlk, extracted);
                          if (rootIndex)
                              rootIndex->mergeSegmentMonitors(tlk);
                          tlk->finishSegmentMonitors();
                          tlk->reset();
                          for (;;)
                          {
                              typedef const void * cvp;
                              if (thisKey->isTopLevelKey())
                              {
                                  bool locallySorted = !thisKey->isFullySorted();
                                  while (locallySorted || tlk->lookup(false)) 
                                  {
                                      unsigned agentPart = locallySorted ? tlk->getPartition() : (unsigned)extractFpos(tlk);
                                      if (locallySorted || agentPart)
                                      {
                                          cvp *outputBuffer = (cvp *) remote.getMem(agentPart, fileNo, indexReadSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
                                          *outputBuffer++ = jg;
                                          if (indexReadInputRecordVariable)
                                          {
                                              *(unsigned *) outputBuffer = indexReadSize;
                                              outputBuffer = (cvp*) (((unsigned *) outputBuffer) + 1);
                                          }
                                          jg->notePending();
                                          memcpy(outputBuffer, extracted, indexReadSize);
                                          if (!agentPart)
                                          {
                                              for (unsigned i = 1; i < numChannels; i++)
                                                  jg->notePending();
                                          }
                                          if (locallySorted)
                                              break;
                                      }
                                  }
                              }
                              else
                              {
                                  // MORE - this code seems to be duplicated in half keyed
                                  unsigned accepted = 0;
                                  unsigned rejected = 0;
                                  Owned<CRowArrayMessageResult> result = new CRowArrayMessageResult();
                                  jg->notePending();
                                  unsigned candidateCount = 0;
                                  ScopedAtomic<unsigned> indexRecordsRead(::indexRecordsRead);
                                  ScopedAtomic<unsigned> postFiltered(::postFiltered);
                                  while (tlk->lookup(true))
                                  {
                                      candidateCount++;
                                      indexRecordsRead++;
                                      KLBlobProviderAdapter adapter(tlk);
                                      const byte *indexRow = tlk->queryKeyBuffer();
                                      size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
                                      offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
                                      if (helper.indexReadMatch(extracted, indexRow, &adapter))
                                      {
                                          KeyedJoinHeader *rhs = (KeyedJoinHeader *) ctx->queryRowManager().allocate(KEYEDJOIN_RECORD_SIZE(0), activityId);
                                          rhs->fpos = fpos;
                                          rhs->thisGroup = jg; 
                                          rhs->partNo = partNo; 
                                          result->append(rhs);
                                      }
                                      else
                                      {
                                          rejected++;
                                          postFiltered++;
                                      }
                                  }
                                  // output an end marker for the matches to this group
                                  KeyedJoinHeader *endMarker = (KeyedJoinHeader *) ctx->queryRowManager().allocate(KEYEDJOIN_RECORD_SIZE(0), activityId);
                                  endMarker->fpos = (offset_t) candidateCount;
                                  endMarker->thisGroup = jg;
                                  endMarker->partNo = (unsigned short) -1;
                                  result->append(endMarker);
                                  remote.injectResult(result.getClear());
                                  if (accepted)
                                      noteStatistic(StNumIndexAccepted, accepted);
                                  if (rejected)
                                      noteStatistic(StNumIndexRejected, rejected);
                              }
  
                              if (++fileNo < thisBase->numParts())
                              {
                                  thisKey = thisBase->queryPart(fileNo);
                                  tlk->setKey(thisKey);
                                  if (!thisKey->isTopLevelKey())
                                      tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                                  else
                                      tlk->setLayoutTranslator(nullptr);
                                  tlk->reset();
                              }
                              else
                                  break;
                          }
                          tlk->releaseSegmentMonitors();
                          tlk->setKey(NULL);
                      }
                      catch (...)
                      {
                          tlk->releaseSegmentMonitors();
                          tlk->setKey(NULL);
                          throw;
                      }
                  }
              }
              joinHandler->noteEndReceived(jg, 0);
          }
          else
          {
              joinHandler->noteEndReceived(joinHandler->createJoinGroup(row), 0);
          }
      }
  
      void processGroup(const ConstPointerArray &)
      {
          throwUnexpected();
      }
  
      virtual void processEOG()
      {
          joinHandler->processEOG();
      }
  
      virtual void processDone()
      {
          // called from puller thread
          remote.flush();
          remote.senddone();
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
              throwUnexpected();
          helper.onLimitExceeded();
      }
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          throwUnexpected();
      }
  
      virtual bool fireException(IException *e)
      {
          // called from puller thread on failure
          remote.fireException(LINK(e));
          return joinHandler->fireException(e);
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          remote.gatherStats(merged);
          if (indexReadAllocator)
              indexReadAllocator->gatherStats(merged);
      }
  };
  
  class CRoxieServerKeyedJoinBase : public CRoxieServerActivity, implements IRecordPullerCallback, implements IRoxieServerErrorHandler, implements IJoinProcessor
  {
  protected:
      IHThorKeyedJoinArg &helper;
      KeyedJoinRemoteAdaptor remote;
      RecordPullerThread puller;
      OwnedConstRoxieRow defaultRight;
      Owned<IEngineRowAllocator> defaultRightAllocator;
      unsigned joinFlags;
      unsigned atMost;
      unsigned atmostsTriggered;
      unsigned abortLimit;
      unsigned keepLimit;
      bool limitFail;
      bool limitOnFail;
      bool preserveGroups;
      bool cloneLeft;
      bool isSimple;
      bool isLocal;
      ThorActivityKind activityKind;
      CJoinGroup *groupStart;
      CriticalSection groupsCrit;
      QueueOf<CJoinGroup, false> groups;
      IFinalRoxieInput *indexReadInput;
      unsigned indexReadIdx = 0;
      Owned<IStrandJunction> indexReadJunction;
      IEngineRowStream *indexReadStream = NULL;  // Never actually pulled
      IIndexReadActivityInfo *rootIndex;
  
      void createDefaultRight()
      {
          if (!defaultRight)
          {
              if (!defaultRightAllocator)
                  defaultRightAllocator.setown(createRowAllocator(helper.queryJoinFieldsRecordSize()));
  
              RtlDynamicRowBuilder rowBuilder(defaultRightAllocator);
              size32_t thisSize = helper.createDefaultRight(rowBuilder);
              defaultRight.setown(rowBuilder.finalizeRowClear(thisSize));
          }
      }
  
  public:
      CRoxieServerKeyedJoinBase(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _joinFlags
          , bool isFull, bool _isSimple, bool _isLocal)
          : CRoxieServerActivity(_ctx, _factory, _probeManager),
            helper((IHThorKeyedJoinArg &)basehelper), 
            remote(_ctx, this, _remoteId, helper, *this, isFull, _isSimple, puller, *this),
            puller(false),
            joinFlags(_joinFlags), 
            atMost(0),
            abortLimit(0),
            keepLimit(0),
            limitFail(false),
            limitOnFail(false),
            preserveGroups(meta.isGrouped()),
            cloneLeft(false),
            isSimple(_isSimple),
            isLocal(_isLocal)
      {
          groupStart = NULL;
          activityKind = _factory->getKind();
          indexReadInput = NULL;
          rootIndex = NULL;
          atmostsTriggered = 0;
          // MORE - code would be easier to read if I got more values from helper rather than passing from factory
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerActivity::onCreate(_colocalParent);
          remote.onCreate(_colocalParent);
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          if (!idx)
              puller.setInput(this, _sourceIdx, _in);
          else if (idx==1)
          {
              indexReadInput = _in;
              indexReadIdx = _sourceIdx;
          }
          else
              throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() parameter out of bounds at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__);
      }
  
      virtual void connectInputStreams(bool consumerOrdered)
      {
          CRoxieServerActivity::connectInputStreams(consumerOrdered);
          puller.connectInputStreams(ctx, consumerOrdered);
          if (indexReadInput)
              indexReadStream = connectSingleStream(ctx, indexReadInput, indexReadIdx, indexReadJunction, consumerOrdered);  // We never actually pull the stream
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          if (indexReadInput)
          {
              indexReadInput->start(parentExtractSize, parentExtract, true); // paused=true because we don't want to actually run the index read
              rootIndex = indexReadInput->queryIndexReadActivity();
              if (!rootIndex)
                  throw MakeStringException(ROXIE_INTERNAL_ERROR,"Index in keyed join %d could not be resolved", queryId());
          }
          remote.onStart(parentExtractSize, parentExtract);
          remote.setLimits(helper.getRowLimit(), (unsigned __int64) -1, I64C(0x7FFFFFFFFFFFFFFF));
          atmostsTriggered = 0;
          atMost = helper.getJoinLimit();
          if (atMost == 0) atMost = (unsigned)-1;
          abortLimit = helper.getMatchAbortLimit();
          if (abortLimit == 0 || atMost != (unsigned) -1) abortLimit = (unsigned)-1;
          keepLimit = helper.getKeepLimit();
          if (keepLimit == 0) keepLimit = (unsigned)-1;
          getLimitType(joinFlags, limitFail, limitOnFail);
          cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
          if ((joinFlags & JFleftouter) || limitOnFail)
              createDefaultRight();
      }
  
      virtual void stop()
      {
          puller.stop();
          if (indexReadStream)
              indexReadStream->stop();   // Could probably do this as soon as re have fetched rootIndex?
          CRoxieServerActivity::stop();
      }
  
      virtual unsigned __int64 queryLocalCycles() const
      {
          __int64 localCycles = remote.activityStats.totalCycles;
          localCycles -= puller.queryTotalCycles(); // MORE - debatable... but probably fair.
          if (localCycles < 0)
              localCycles = 0;
          return localCycles;
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerActivity::gatherStats(merged);
          remote.gatherStats(merged);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          if (idx==0)
              return puller.queryInput();
          else if (idx==1)
              return indexReadInput;
          else
              return NULL;
      }
  
      virtual void reset()
      {
          activityStats.totalCycles = remote.activityStats.totalCycles;
          remote.activityStats.totalCycles = 0;
          processed = remote.joinProcessed;
          remote.joinProcessed = 0;
          defaultRight.clear();
          if (indexReadInput)
              indexReadInput->reset();
          if (atmostsTriggered)
              noteStatistic(StNumAtmostTriggered, atmostsTriggered);
          CRoxieServerActivity::reset(); 
          puller.reset();
          while (groups.ordinality())
          {
              ::Release(groups.dequeue());
          }
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          if (idx==(unsigned)-1)
              idx = 0;
          return idx ? NULL : &remote;
      }
  
      virtual CJoinGroup *createJoinGroup(const void *row)
      {
          // NOTE - we need to protect access to queue, since it's also modified by consumer thread. Groupstart is only modified by puller thread.
          CriticalBlock c(groupsCrit);
          if (preserveGroups && !groupStart)
          {
              groupStart = new (&ctx->queryRowManager(), activityId) CJoinGroup(NULL,  NULL);
              groups.enqueue(groupStart);
          }
          CJoinGroup *jg = new (&ctx->queryRowManager(), activityId) CJoinGroup(row, groupStart);
          groups.enqueue(jg);
          return jg;
      }
  
      void endGroup()
      {
          CriticalBlock c(groupsCrit);
          if (groupStart)
              noteEndReceived(groupStart, 0);
          groupStart = NULL;
      }
  
      virtual void noteEndReceived(CJoinGroup *jg, unsigned candidateCount)
      {
          if (jg->noteEndReceived(candidateCount))
              processCompletedGroups();
      }
  
      void processCompletedGroups()
      {
          CriticalBlock c(groupsCrit);
          while (groups.ordinality())
          {
              if (!groups.head()->complete())
                  break;
              Owned<CJoinGroup> head = groups.dequeue();
              if (preserveGroups)
              {
                  assert(head->isHeadRecord());
                  assert(groups.head()->inGroup(head));
                  unsigned joinGroupSize = 0;
                  while (groups.ordinality() && groups.head()->inGroup(head))
                  {
                      Owned<CJoinGroup> finger = groups.dequeue();
                      joinGroupSize += doJoinGroup(finger);
                  }
                  if (joinGroupSize)
                      remote.addResult(NULL);
              }
              else
                  doJoinGroup(head);
          }
      }
  
      void failLimit(const void * left)
      {
          helper.onMatchAbortLimitExceeded();
          CommonXmlWriter xmlwrite(0);
          if (input && input->queryOutputMeta() && input->queryOutputMeta()->hasXML())
          {
              input->queryOutputMeta()->toXML((byte *) left, xmlwrite);
          }
          throw MakeStringException(ROXIE_JOIN_ERROR, "More than %d match candidates in keyed join %d for row %s", abortLimit, queryId(), xmlwrite.str());
      }
  
      virtual bool needsAllocator() const { return true; }
  
      unsigned doTransform(const void *left, const void *right, offset_t fpos_or_count, IException *except, const void **group, unsigned counter)
      {
          if (cloneLeft && !except)
          {
              LinkRoxieRow(left);
              remote.addResult((void *) left);
              return 1;
          }
          
          RtlDynamicRowBuilder rowBuilder(rowAllocator);
          unsigned outSize;
          try
          {   
              outSize = except ? helper.onFailTransform(rowBuilder, left, right, fpos_or_count, except) :
                        (activityKind == TAKkeyeddenormalizegroup) ? helper.transform(rowBuilder, left, right, (unsigned)fpos_or_count, group) :
                        helper.transform(rowBuilder, left, right, fpos_or_count, counter);
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
          if (outSize)
          {
              const void *shrunk = rowBuilder.finalizeRowClear(outSize);
              remote.addResult(shrunk);
              return 1;
          }
          else
              return 0;
      }
  
      unsigned doJoinGroup(CJoinGroup *jg)
      {
          unsigned matched = jg->rowsSeen();
          unsigned added = 0;
          const void *left = jg->queryLeft();
  
          if (jg->candidateCount() > abortLimit)
          {
              if (limitFail)
                  failLimit(left);
              if (ctx->queryDebugContext())
                  ctx->queryDebugContext()->checkBreakpoint(DebugStateLimit, NULL, static_cast<IActivityBase *>(this));
              if (limitOnFail)
              {
                  Owned<IException> except;
                  try
                  {
                      failLimit(left);
                  }
                  catch(IException * e)
                  {
                      except.setown(e);
                  }
                  added = doTransform(left, defaultRight, 0, except, NULL, 0);
              }
          }
          else if (!matched || jg->candidateCount() > atMost)
          {
              if (jg->candidateCount() > atMost)
                  atmostsTriggered++;
              switch (joinFlags & JFtypemask)
              {
              case JFleftouter:
              case JFleftonly:
                  switch (activityKind)
                  {
                  case TAKkeyedjoin:
                  case TAKkeyeddenormalizegroup:
                      added = doTransform(left, defaultRight, 0, NULL, NULL, 0);
                      break;
                  case TAKkeyeddenormalize:
                      LinkRoxieRow(left);
                      remote.addResult((void *) left);
                      added++;
                      break;
                  }
                  break;
              }
          }
          else if (!(joinFlags & JFexclude))
          {
              unsigned idx = 0;
              switch (activityKind)
              {
              case TAKkeyedjoin:
                  while (idx < matched)
                  {
                      const KeyedJoinHeader *rhs = jg->queryRow(idx);
                      added += doTransform(left, &rhs->rhsdata, 0, NULL, NULL, idx+1);
                      if (added==keepLimit)
                          break;
                      idx++;
                  }
                  break;
              case TAKkeyeddenormalize:
                  {
                      OwnedConstRoxieRow newLeft;
                      newLeft.set(left);
                      unsigned rowSize = 0;
                      unsigned rightAdded = 0;
                      while (idx < matched)
                      {
                          const KeyedJoinHeader *rhs = jg->queryRow(idx);
                          try
                          {
                              RtlDynamicRowBuilder rowBuilder(rowAllocator);
                              size32_t transformedSize = helper.transform(rowBuilder, newLeft, &rhs->rhsdata, rhs->fpos, idx+1);
                              if (transformedSize)
                              {
                                  rowSize = transformedSize;
                                  newLeft.setown(rowBuilder.finalizeRowClear(rowSize));
                                  rightAdded++;
                                  if (rightAdded==keepLimit)
                                      break;
                              }
                              idx++;
                          }
                          catch (IException *E)
                          {
                              throw makeWrappedException(E);
                          }
                      }
                      if (rowSize)
                      {
                          remote.addResult(newLeft.getClear());
                          added++;
                      }
                  }
                  break;
              case TAKkeyeddenormalizegroup:
                  {
                      ConstPointerArray extractedRows;
                      while (idx < matched && idx < keepLimit)
                      {
                          const KeyedJoinHeader *rhs = jg->queryRow(idx);
                          extractedRows.append((void *) &rhs->rhsdata);
                          idx++;
                      }
                      added += doTransform(left, extractedRows.item(0), extractedRows.ordinality(), NULL, (const void * *)extractedRows.getArray(), 0);
                  }
                  break;
              }
          }
          return added;
      }
  
      virtual void processDone()
      {
          // called from puller thread
          remote.flush();
          remote.senddone();
      }
  
      virtual void onLimitExceeded(bool isKeyed)
      {
          if (traceLevel > 4)
              DBGLOG("activityid = %d  isKeyed = %d  line = %d", activityId, isKeyed, __LINE__);
          if (isKeyed)
              throwUnexpected();
          helper.onLimitExceeded();
      }
      virtual const void *createLimitFailRow(bool isKeyed)
      {
          throwUnexpected();
      }
  
      virtual bool fireException(IException *e)
      {
          // called from puller thread on failure
          return remote.fireException(e);
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  };
  
  #ifdef _MSC_VER
  #pragma warning ( push )
  #pragma warning ( disable: 4355 )
  #endif
  
  class CRoxieServerKeyedJoinActivity : public CRoxieServerKeyedJoinBase 
  {
      CRoxieServerFullKeyedJoinHead head;
      Owned<IEngineRowAllocator> fetchInputAllocator;
      Linked<IFilePartMap> map;
      bool variableFetchFileName;
      Owned<const IResolvedFile> varFetchFileInfo;
      CachedOutputMetaData fetchInputFields;
  
  public:
  
      CRoxieServerKeyedJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_headId, IKeyArray * _key, ITranslatorSet *_keyTranslators, IOutputMetaData *_indexReadMeta,
          const RemoteActivityId &_tailId, IFilePartMap *_map, unsigned _joinFlags, bool _isLocal)
          : CRoxieServerKeyedJoinBase(_ctx, _factory, _probeManager, _tailId, _joinFlags, true, false, _isLocal),
            head(_ctx, _factory, _probeManager, _headId, _key, _keyTranslators, _indexReadMeta, this, _isLocal),
            map(_map)
      {
          CRoxieServerKeyedJoinBase::setInput(0, 0, head.queryOutput(0));
          variableFetchFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
      }
      
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFetchFileInfo;
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerKeyedJoinBase::onCreate(_colocalParent);
          head.onCreate(_colocalParent);
          fetchInputFields.set(helper.queryFetchInputRecordSize());
          fetchInputAllocator.setown(createRowAllocator(helper.queryFetchInputRecordSize()));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerKeyedJoinBase::start(parentExtractSize, parentExtract, paused);
          if (variableFetchFileName)
          {
              bool isFetchOpt = (helper.getFetchFlags() & FFdatafileoptional) != 0;
              OwnedRoxieString fname(helper.getFileName());
              varFetchFileInfo.setown(resolveLFNFlat(fname, isFetchOpt, factory->isActivityCodeSigned()));
              if (varFetchFileInfo)
              {
                  // Note - we don't need to do any translation on the disk part of full-keyed joins
                  map.setown(varFetchFileInfo->getFileMap());
              }
          }
          puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().keyedJoinPreload, false, ctx);
      }
  
      virtual void setInput(unsigned idx, unsigned _sourceIdx, IFinalRoxieInput *_in)
      {
          head.setInput(idx, _sourceIdx, _in);
      }
  
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          connectInputStreams(consumerOrdered);
          return head.getOutputStreams(ctx, idx, streams, consumerOptions, consumerOrdered, orderedCallbacks);
      }
  
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          return head.queryInput(idx);
      }
  
      virtual void processRow(const void *_rhs)
      {
          // called from puller thread
          KeyedJoinHeader *rhs = (KeyedJoinHeader *) _rhs;
          CJoinGroup *jg = rhs->thisGroup;
          if (rhs->partNo != (unsigned short) -1)
          {
              unsigned partNo = map->mapOffset(rhs->fpos);
              RtlDynamicRowBuilder fetchBuilder(fetchInputAllocator, true);
              size32_t fisize = helper.extractFetchFields(fetchBuilder, jg->queryLeft());
              if (fetchInputFields.isVariableSize())
              {
                  KeyedJoinHeader *outRow = (KeyedJoinHeader *) remote.getMem(partNo, 0, KEYEDJOIN_RECORD_SIZE(fisize + sizeof(fisize)));
                  memcpy(outRow, rhs, KEYEDJOIN_RECORD_SIZE(0)); // MORE - copy constructor might be more appropriate....
                  ReleaseRoxieRow(rhs);
                  jg->notePending();
                  memcpy(&outRow->rhsdata, &fisize, sizeof(fisize));
                  memcpy((&outRow->rhsdata)+sizeof(fisize), fetchBuilder.row(), fisize);
              }
              else
              {
                  KeyedJoinHeader *outRow = (KeyedJoinHeader *) remote.getMem(partNo, 0, KEYEDJOIN_RECORD_SIZE(fisize));
                  memcpy(outRow, rhs, KEYEDJOIN_RECORD_SIZE(0)); // MORE - copy constructor might be more appropriate....
                  ReleaseRoxieRow(rhs);
                  jg->notePending();
                  memcpy(&outRow->rhsdata, fetchBuilder.row(), fisize);
              }
          }
          else
          {
              unsigned candidateCount = (unsigned) rhs->fpos;
  //          CTXLOG("Full keyed join - all results back from index");
              ReleaseRoxieRow(rhs);
              noteEndReceived(jg, candidateCount); // may throw exception - so release row before calling
          }
      }
  
      virtual void processEOG()
      {
          // called from front puller thread
          if (preserveGroups)
              endGroup();
      }
  
      void processGroup(const ConstPointerArray &)
      {
          throwUnexpected();
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerKeyedJoinBase::gatherStats(merged);
          if (fetchInputAllocator)
              fetchInputAllocator->gatherStats(merged);
      }
  };
  
  #ifdef _MSC_VER
  #pragma warning ( pop )
  #endif
  
  class CRoxieServerHalfKeyedJoinActivity : public CRoxieServerKeyedJoinBase 
  {
      IOutputMetaData *indexReadMeta;
      Owned<IEngineRowAllocator> indexReadAllocator;
      Owned<IKeyManager> tlk;
      bool variableIndexFileName;
      bool indexReadInputRecordVariable;
      Owned<const IResolvedFile> varFileInfo;
      Linked<ITranslatorSet> translators;
      Linked<IKeyArray> keySet;
      Owned<IOutputMetaData> joinPrefixedMeta;
      Owned<IEngineRowAllocator> joinFieldsAllocator;
  
  public:
      CRoxieServerHalfKeyedJoinActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, IKeyArray * _keySet, ITranslatorSet *_translators,
          IOutputMetaData *_indexReadMeta, unsigned _joinFlags, bool _isSimple, bool _isLocal)
          : CRoxieServerKeyedJoinBase(_ctx, _factory, _probeManager, _remoteId, _joinFlags, false, _isSimple, _isLocal),
            indexReadMeta(_indexReadMeta),
            tlk(createLocalKeyManager(helper.queryIndexRecordSize()->queryRecordAccessor(true), NULL, this, helper.hasNewSegmentMonitors(), !isBlind())),
            translators(_translators),
            keySet(_keySet)
      {
          variableIndexFileName = allFilesDynamic || factory->queryQueryFactory().isDynamic() || ((helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
          indexReadInputRecordVariable = indexReadMeta->isVariableSize();
      }
  
      virtual void serializeExtra(MemoryBuffer &out)
      {
          if (helper.getJoinFlags() & JFindexfromactivity)
          {
              assertex(rootIndex);
              const RemoteActivityId& indexId = rootIndex->queryRemoteId();
              indexId.serialize(out);
              // could mess about reserving space for length then patching it again, to avoid copy, but probably not worth it
              MemoryBuffer tmp;
              rootIndex->queryActivity()->serializeCreateStartContext(tmp);
              if (rootIndex->queryActivity()->queryVarFileInfo())
              {
                  rootIndex->queryActivity()->queryVarFileInfo()->queryTimeStamp().serialize(tmp);
                  tmp.append(rootIndex->queryActivity()->queryVarFileInfo()->queryCheckSum());
              }
              unsigned ctxlen = tmp.length();
              out.append(ctxlen).append(tmp);
          }
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          CRoxieServerKeyedJoinBase::onCreate(_colocalParent);
          indexReadAllocator.setown(createRowAllocator(indexReadMeta));
  
          IOutputMetaData *joinFieldsMeta = helper.queryJoinFieldsRecordSize();
          joinPrefixedMeta.setown(new CPrefixedOutputMeta(KEYEDJOIN_RECORD_SIZE(0), joinFieldsMeta)); // MORE - not sure if we really need this
          joinFieldsAllocator.setown(createRowAllocator(joinPrefixedMeta));
      }
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          CRoxieServerKeyedJoinBase::start(parentExtractSize, parentExtract, paused);
          if (rootIndex)
          {
              varFileInfo.setown(rootIndex->getVarFileInfo());
              translators.setown(rootIndex->getTranslators());
              keySet.setown(rootIndex->getKeySet());
          }
          else if (variableIndexFileName)
          {
              OwnedRoxieString indexFileName(helper.getIndexFileName());
              varFileInfo.setown(resolveLFNIndex(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0, factory->isActivityCodeSigned()));
              if (varFileInfo)
              {
                  unsigned expectedCrc = helper.getIndexFormatCrc();
                  unsigned projectedCrc = helper.getProjectedIndexFormatCrc();
                  translators.setown(varFileInfo->getTranslators(projectedCrc, helper.queryProjectedIndexRecordSize(), expectedCrc, helper.queryIndexRecordSize(), getEnableFieldTranslation(), FileFormatMode::index, factory->queryQueryFactory().queryQueryName()));
                  keySet.setown(varFileInfo->getKeyArray(false, isLocal ? factory->queryQueryFactory().queryChannel() : 0));
              }
          }
          puller.start(parentExtractSize, parentExtract, paused, ctx->queryOptions().keyedJoinPreload, isSimple, ctx);
  
      }
  
      virtual const IResolvedFile *queryVarFileInfo() const
      {
          return varFileInfo;
      }
  
      virtual void reset()
      {
          CRoxieServerKeyedJoinBase::reset();
          if (varFileInfo)
          {
              keySet.clear();
              varFileInfo.clear();
          }
      }
  
      virtual void processRow(const void *row)
      {
          // called from front puller thread
          // buffer up an IndexRead request
          if (keySet && helper.leftCanMatch(row))
          {
              RtlDynamicRowBuilder extractBuilder(indexReadAllocator);
              unsigned indexReadRecordSize = helper.extractIndexReadFields(extractBuilder, row);
              OwnedConstRoxieRow extracted;
              if (indexReadRecordSize)
                  extracted.setown(extractBuilder.finalizeRowClear(indexReadRecordSize));
  
              CJoinGroup *jg = createJoinGroup(row);
              for (unsigned partNo = 0; partNo < keySet->length(); partNo++)
              {
                  IKeyIndexBase *thisBase = keySet->queryKeyPart(partNo);
                  if (thisBase)
                  {
                      unsigned fileNo = 0;
                      IKeyIndex *thisKey = thisBase->queryPart(fileNo);
                      tlk->setKey(thisKey);
                      if (thisKey && !thisKey->isTopLevelKey())
                          tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                      else
                          tlk->setLayoutTranslator(nullptr);
                      helper.createSegmentMonitors(tlk, extracted);
                      if (rootIndex)
                          rootIndex->mergeSegmentMonitors(tlk);
                      tlk->finishSegmentMonitors();
                      try
                      {
                          tlk->reset();
                          for (;;)
                          {
                              typedef const void * cvp;
                              if (thisKey && thisKey->isTopLevelKey())
                              {
                                  bool locallySorted = (!thisKey->isFullySorted());
                                  while (locallySorted || tlk->lookup(false))
                                  {
                                      unsigned agentPart = locallySorted ? tlk->getPartition() : (unsigned)extractFpos(tlk);
                                      if (locallySorted || agentPart)
                                      {
                                          cvp *outputBuffer = (cvp *) remote.getMem(agentPart, fileNo, indexReadRecordSize + sizeof(cvp) + (indexReadInputRecordVariable ? sizeof(unsigned) : 0));
                                          *outputBuffer++ = jg;
                                          if (indexReadInputRecordVariable)
                                          {
                                              *(unsigned *) outputBuffer = indexReadRecordSize;
                                              outputBuffer = (cvp *) (((unsigned *) outputBuffer) + 1);
                                          }
                                          jg->notePending();
                                          memcpy(outputBuffer, extracted, indexReadRecordSize);
                                          if (!agentPart)
                                          {
                                              for (unsigned i = 1; i < numChannels; i++)
                                                  jg->notePending();
                                          }
                                          if (locallySorted)
                                              break;
                                      }
                                  }
                              }
                              else
                              {
                                  unsigned accepted = 0;
                                  unsigned rejected = 0;
                                  Owned<CRowArrayMessageResult> result;
                                  if (!isSimple) 
                                      result.setown(new CRowArrayMessageResult());
                                  // MORE - This code seems to be duplicated in keyedJoinHead
                                  jg->notePending();
                                  unsigned candidateCount = 0;
                                  ScopedAtomic<unsigned> indexRecordsRead(::indexRecordsRead);
                                  ScopedAtomic<unsigned> postFiltered(::postFiltered);
                                  while (tlk->lookup(true))
                                  {
                                      candidateCount++;
                                      indexRecordsRead++;
                                      KLBlobProviderAdapter adapter(tlk);
                                      const byte *indexRow = tlk->queryKeyBuffer();
                                      size_t fposOffset = tlk->queryRowSize() - sizeof(offset_t);
                                      offset_t fpos = rtlReadBigUInt8(indexRow + fposOffset);
                                      if (helper.indexReadMatch(extracted, indexRow, &adapter))
                                      {
                                          RtlDynamicRowBuilder rb(joinFieldsAllocator, true); 
                                          CPrefixedRowBuilder pb(KEYEDJOIN_RECORD_SIZE(0), rb);
                                          accepted++;
                                          KLBlobProviderAdapter adapter(tlk);
                                          helper.extractJoinFields(pb, indexRow, &adapter);
                                          KeyedJoinHeader *rec = (KeyedJoinHeader *) rb.getUnfinalizedClear(); // lack of finalize ok as unserialized data here.
                                          rec->fpos = fpos;
                                          rec->thisGroup = jg;
                                          rec->partNo = partNo;
                                          if (isSimple)
                                              remote.injected.enqueue(rec);
                                          else
                                              result->append(rec);
                                      }
                                      else
                                      {
                                          rejected++;
                                          postFiltered++;
                                      }
                                  }
  
                                  // output an end marker for the matches to this group
                                  KeyedJoinHeader *rec = (KeyedJoinHeader *) ctx->queryRowManager().allocate(KEYEDJOIN_RECORD_SIZE(0), activityId);
                                  rec->fpos = (offset_t) candidateCount;
                                  rec->thisGroup = jg;
                                  rec->partNo = (unsigned short) -1;
                                  if (isSimple)
                                      remote.injected.enqueue(rec);
                                  else
                                  {
                                      result->append(rec);
                                      remote.injectResult(result.getClear());
                                  }
                                  if (accepted)
                                      noteStatistic(StNumIndexAccepted, accepted);
                                  if (rejected)
                                      noteStatistic(StNumIndexRejected, rejected);
                              }
                              if (++fileNo < thisBase->numParts())
                              {
                                  thisKey = thisBase->queryPart(fileNo);
                                  tlk->setKey(thisKey);
                                  if (thisKey && !thisKey->isTopLevelKey())
                                      tlk->setLayoutTranslator(translators->queryTranslator(fileNo));
                                  else
                                      tlk->setLayoutTranslator(nullptr);
                                  tlk->reset();
                              }
                              else
                                  break;
                          }
                          tlk->releaseSegmentMonitors();
                          tlk->setKey(NULL);
                      }
                      catch (...)
                      {
                          tlk->releaseSegmentMonitors();
                          tlk->setKey(NULL);
                          throw;
                      }
                  }
              }
              noteEndReceived(jg, 0);
          }
          else
          {
              noteEndReceived(createJoinGroup(row), 0);
          }
      }
  
      virtual void processEOG()
      {
          // called from front puller thread
          if (preserveGroups)
              endGroup();
      }
  
      void processGroup(const ConstPointerArray &)
      {
          throwUnexpected();
      }
  
      virtual void gatherStats(CRuntimeStatisticCollection & merged) const override
      {
          CRoxieServerKeyedJoinBase::gatherStats(merged);
          if (indexReadAllocator)
              indexReadAllocator->gatherStats(merged);
          if (joinFieldsAllocator)
              joinFieldsAllocator->gatherStats(merged);
      }
  };
  
  class CRoxieServerKeyedJoinActivityFactory : public CRoxieServerMultiInputFactory
  {
      Owned<const IResolvedFile> indexfile;
      Owned<const IResolvedFile> datafile;
      Owned<IKeyArray> keySet;
      Owned<ITranslatorSet> keyTranslators;  // For the indexes
      RemoteActivityId headId;
      RemoteActivityId tailId;
      IOutputMetaData *indexReadMeta;
      Owned<IFilePartMap> map;
      Owned<ITranslatorSet> translators;  // for the disk files on local full keyed joins
      Owned<IFileIOArray> files;
      unsigned joinFlags;
      bool isHalfKeyed;
      bool isLocal;
      bool variableFetchFileName;
      bool variableIndexFileName;
      bool isSimple;
  
  public:
      CRoxieServerKeyedJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_headId, const RemoteActivityId &_tailId)
          : CRoxieServerMultiInputFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, keyedJoinStatistics), headId(_headId), tailId(_tailId)
      {
          Owned<IHThorKeyedJoinArg> helper = (IHThorKeyedJoinArg *) helperFactory();
          isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
          isSimple = isLocal;
          rtlDataAttr indexLayoutMeta;
          size32_t indexLayoutSize = 0;
          if(!helper->getIndexLayout(indexLayoutSize, indexLayoutMeta.refdata()))
              assertex(indexLayoutSize== 0);
          joinFlags = helper->getJoinFlags();
          variableIndexFileName = allFilesDynamic || _queryFactory.isDynamic() || ((joinFlags & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
          variableFetchFileName = allFilesDynamic || _queryFactory.isDynamic() || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
          if (!variableIndexFileName)
          {
              bool isOpt = (joinFlags & JFindexoptional) != 0;
              OwnedRoxieString indexFileName(helper->getIndexFileName());
              indexfile.setown(queryFactory.queryPackage().lookupFileName(indexFileName, isOpt, true, true, queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
              if (indexfile)
              {
                  unsigned expectedCrc = helper->getIndexFormatCrc();
                  unsigned projectedCrc = helper->getProjectedIndexFormatCrc();
                  keyTranslators.setown(indexfile->getTranslators(projectedCrc, helper->queryProjectedIndexRecordSize(), expectedCrc, helper->queryIndexRecordSize(), getEnableFieldTranslation(), FileFormatMode::index, queryFactory.queryQueryName()));
                  keySet.setown(indexfile->getKeyArray(isOpt, isLocal ? queryFactory.queryChannel() : 0));
              }
          }
          if (keySet && keySet->length()==1 && !isSimple)
          {
              IKeyIndexBase *thisBase = keySet->queryKeyPart(0);
              if (thisBase->numParts()==1 && !thisBase->queryPart(0)->isTopLevelKey() && !_queryFactory.queryOptions().disableLocalOptimizations)
                  isSimple = true;
              // MORE - if it's a variable filename then it MAY be simple, we don't know. Tough.
          }
          if (!simpleLocalKeyedJoins)
              isSimple = false;
          isHalfKeyed = !helper->diskAccessRequired();
  
          indexReadMeta = QUERYINTERFACE(helper->queryIndexReadInputRecordSize(), IOutputMetaData);
          if (!isHalfKeyed && !variableFetchFileName)
          {
              bool isFetchOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
              datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode, _kind), isFetchOpt, true, true, _queryFactory.queryWorkUnit(), true, isActivityCodeSigned()));
              if (datafile)
              {
                  if (isLocal)  // Not sure this works
                  {
                      unsigned expectedCrc = helper->getDiskFormatCrc();
                      unsigned projectedCrc = helper->getProjectedFormatCrc();
                      translators.setown(datafile->getTranslators(expectedCrc, helper->queryProjectedDiskRecordSize(), projectedCrc, helper->queryDiskRecordSize(), getEnableFieldTranslation(), FileFormatMode::flat, queryFactory.queryQueryName()));
                      files.setown(datafile->getIFileIOArray(isFetchOpt, queryFactory.queryChannel()));
                  }
                  else
                      map.setown(datafile->getFileMap());
              }
          }
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          if (isHalfKeyed)
              return new CRoxieServerHalfKeyedJoinActivity(_ctx, this, _probeManager,
                  headId, keySet, keyTranslators, indexReadMeta, joinFlags, isSimple, isLocal);
          else
              return new CRoxieServerKeyedJoinActivity(_ctx, this, _probeManager,
                  headId, keySet, keyTranslators, indexReadMeta,
                  tailId, map, joinFlags, isLocal); // MORE - not sure local variant actually works - should be using files not map, for example?
      }
  
      virtual void getXrefInfo(IPropertyTree &reply, const IRoxieContextLogger &logctx) const
      {
          if (datafile)
              addXrefFileInfo(reply, datafile);
          if (indexfile)
              addXrefFileInfo(reply, indexfile);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerKeyedJoinActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, const RemoteActivityId &_remoteId, const RemoteActivityId &_remoteId2)
  {
      return new CRoxieServerKeyedJoinActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _remoteId, _remoteId2);
  }
  
  //=================================================================================
  
  class CRoxieServerSoapActivityBase : public CRoxieServerActivity, implements ISoapCallRowProvider, implements IRoxieAbortMonitor
  {
  protected:
      Owned<ISoapCallHelper> soaphelper;
      IHThorSoapActionArg & helper;
      StringBuffer authToken;
      bool eof;
      CriticalSection crit;
      ClientCertificate *pClientCert;
  
  public:
      IMPLEMENT_IINTERFACE_USING(CRoxieServerActivity)
  
      CRoxieServerSoapActivityBase(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerActivity(_ctx, _factory, _probeManager), helper((IHThorSoapActionArg &)basehelper)
      {
          eof = false;
          if (clientCert.certificate.length() > 0 && clientCert.privateKey.length() > 0 && clientCert.passphrase.length() > 0)
              pClientCert = &clientCert;
          else
              pClientCert = NULL;
      }
  
      // ISoapCallRowProvider
      virtual IHThorSoapActionArg * queryActionHelper() { return &helper; };
      virtual IHThorSoapCallArg * queryCallHelper() { return NULL; };
      virtual const void * getNextRow() { return NULL; };
      virtual void releaseRow(const void * r) { ReleaseRoxieRow(r); };
  
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          eof = false;
          CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
          authToken.append(ctx->queryAuthToken());
      }
      virtual void reset()
      {
          // MORE - Shouldn't we make sure thread is stopped etc???
          soaphelper.clear();
          CRoxieServerActivity::reset();
      }
  
      // IRoxieAbortMonitor
      virtual void checkForAbort() { checkAbort(); }
  };
  
  class CRoxieServerSoapActionBase : public CRoxieServerSoapActivityBase
  {
  public:
      CRoxieServerSoapActionBase(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerSoapActivityBase(_ctx, _factory, _probeManager)
      {
      }
  
      virtual void execute(unsigned parentExtractSize, const byte * parentExtract)
      {
          CriticalBlock b(ecrit);
          if (exception)
              throw(exception.getLink());
          if (!executed)
          {
              try
              {
                  executed = true;
                  start(parentExtractSize, parentExtract, false);
                  {
                      ActivityTimer t(activityStats, timeActivities); // unfortunately this is not really best place for seeing in debugger.
                      onExecute();
                  }
                  stop();
              }
              catch (IException *E)
              {
                  E = makeWrappedException(E);
                  ctx->notifyAbort(E);
                  exception.set(E);
                  abort();
                  throw E;
              }
          }
      }
  
      virtual void onExecute() = 0;
  
      virtual void reset() override
      {
          executed = false;
          exception.clear();
          CRoxieServerSoapActivityBase::reset();
      }
  
  protected:
      bool executed = false;
      Linked<IException> exception;
      CriticalSection ecrit;
  };
  //---------------------------------------------------------------------------
  
  class CRoxieServerSoapRowCallActivity : public CRoxieServerSoapActivityBase
  {
      IHThorSoapCallArg & callHelper;
  
  public:
      CRoxieServerSoapRowCallActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerSoapActivityBase(_ctx, _factory, _probeManager), callHelper((IHThorSoapCallArg &)basehelper)
      {
      }
  
      virtual IHThorSoapCallArg * queryCallHelper()
      {
          return &callHelper;
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if(eof) return NULL;
  
          try
          {
              if (soaphelper == NULL)
              {
                  if (factory->getKind()==TAKhttp_rowdataset)
                      soaphelper.setown(createHttpCallHelper(this, rowAllocator, authToken.str(), SCrow, pClientCert, *this, this));
                  else
                      soaphelper.setown(createSoapCallHelper(this, rowAllocator, authToken.str(), SCrow, pClientCert, *this, this));
                  soaphelper->start();
              }
  
              OwnedConstRoxieRow ret = soaphelper->getRow();
              if (!ret)
              {
                  eof = true;
                  return NULL;
              }
              ++processed;
              return ret.getClear();
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  };
  
  class CRoxieServerSoapRowCallActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSoapRowCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, soapStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSoapRowCallActivity(_ctx, this, _probeManager);
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSoapRowCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSoapRowCallActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //---------------------------------------------------------------------------
  
  class CRoxieServerSoapRowActionActivity : public CRoxieServerSoapActionBase
  {
  public:
      CRoxieServerSoapRowActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerSoapActionBase(_ctx, _factory, _probeManager)
      {}
  
      virtual void onExecute() override
      {
          soaphelper.setown(createSoapCallHelper(this, NULL, ctx->queryAuthToken(), SCrow, pClientCert, *this, this));
          soaphelper->start();
          soaphelper->waitUntilDone();
          IException *e = soaphelper->getError();
          soaphelper.clear();
          if (e)
              throw e;
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          return NULL;
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  };
  
  class CRoxieServerSoapRowActionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerSoapRowActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, soapStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSoapRowActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSoapRowActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerSoapRowActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //---------------------------------------------------------------------------
  
  class CRoxieServerSoapDatasetCallActivity : public CRoxieServerSoapActivityBase 
  {
      IHThorSoapCallArg & callHelper;
  
  public:
      CRoxieServerSoapDatasetCallActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerSoapActivityBase(_ctx, _factory, _probeManager), callHelper((IHThorSoapCallArg &)basehelper)
      {
      }
  
      virtual IHThorSoapCallArg * queryCallHelper()
      {
          return &callHelper;
      }
  
      virtual const void *getNextRow()
      {
          CriticalBlock b(crit); // MORE - why ?
          return inputStream->ungroupedNextRow();
      }
  
      virtual bool needsAllocator() const { return true; }
  
      virtual const void *nextRow()
      {
          ActivityTimer t(activityStats, timeActivities);
          if(eof) return NULL;
  
          try
          {
              if (soaphelper == NULL)
              {
                  soaphelper.setown(createSoapCallHelper(this, rowAllocator, authToken.str(), SCdataset, pClientCert, *this, this));
                  soaphelper->start();
              }
  
              OwnedConstRoxieRow ret = soaphelper->getRow();
              if (!ret)
              {
                  eof = true;
                  return NULL;
              }
              ++processed;
              return ret.getClear();
          }
          catch (IException *E)
          {
              throw makeWrappedException(E);
          }
      }
  };
  
  class CRoxieServerSoapDatasetCallActivityFactory : public CRoxieServerActivityFactory
  {
  public:
      CRoxieServerSoapDatasetCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, soapStatistics)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSoapDatasetCallActivity(_ctx, this, _probeManager);
      }
  
  };
  
  IRoxieServerActivityFactory *createRoxieServerSoapDatasetCallActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode)
  {
      return new CRoxieServerSoapDatasetCallActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode);
  }
  
  //---------------------------------------------------------------------------
  
  class CRoxieServerSoapDatasetActionActivity : public CRoxieServerSoapActionBase
  {
  public:
      CRoxieServerSoapDatasetActionActivity(IRoxieAgentContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
          : CRoxieServerSoapActionBase(_ctx, _factory, _probeManager)
      {}
  
      virtual const void *getNextRow()
      {
          CriticalBlock b(crit); // MORE - Why?
          const void *nextrec = inputStream->ungroupedNextRow();
          if (nextrec)
              processed++;
          return nextrec;
      }
  
      virtual void onExecute() override
      {
          soaphelper.setown(createSoapCallHelper(this, NULL, ctx->queryAuthToken(), SCdataset, pClientCert, *this, this));
          soaphelper->start();
          soaphelper->waitUntilDone();
          IException *e = soaphelper->getError();
          soaphelper.clear();
          if (e)
              throw e;
      }
  
      virtual IFinalRoxieInput *queryOutput(unsigned idx)
      {
          return NULL;
      }
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  };
  
  class CRoxieServerSoapDatasetActionActivityFactory : public CRoxieServerActivityFactory
  {
      bool isRoot;
  public:
      CRoxieServerSoapDatasetActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
          : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, soapStatistics), isRoot(_isRoot)
      {
      }
  
      virtual IRoxieServerActivity *createActivity(IRoxieAgentContext *_ctx, IProbeManager *_probeManager) const
      {
          return new CRoxieServerSoapDatasetActionActivity(_ctx, this, _probeManager);
      }
  
      virtual bool isSink() const
      {
          return isRoot;
      }
  };
  
  IRoxieServerActivityFactory *createRoxieServerSoapDatasetActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IPropertyTree &_graphNode, bool _isRoot)
  {
      return new CRoxieServerSoapDatasetActionActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _graphNode, _isRoot);
  }
  
  //=====================================================================================================
  
  class CGraphResults : implements IRoxieGraphResults, public CInterface
  {
      IArrayOf<IGraphResult> results;
      CriticalSection cs;
  
      IGraphResult & select(unsigned idx)
      {
          CriticalBlock procedure(cs);
          if (idx >= results.ordinality())
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Error reading graph result %d before it is calculated", idx);
  
          return results.item(idx);
      }
  
  public:
      IMPLEMENT_IINTERFACE
  
      void clear()
      {
          CriticalBlock procedure(cs);
          results.kill();
      }
  
      virtual IEngineRowStream * createIterator(unsigned id)
      {
          return select(id).createIterator();
      }
      virtual void getLinkedResult(unsigned & count, const byte * * & ret, unsigned id) override
      {
          select(id).getLinkedResult(count, ret);
      }
      virtual void getDictionaryResult(unsigned & count, const byte * * & ret, unsigned id) override
      {
          select(id).getLinkedResult(count, ret);
      }
      virtual const void * getLinkedRowResult(unsigned id)
      {
          return select(id).getLinkedRowResult();
      }
      void setResult(unsigned id, IGraphResult * result)
      {
          CriticalBlock procedure(cs);
  
          if (results.ordinality() <= id)
          {
              while (results.ordinality() < id)
                  results.append(*new CGraphResult);
              results.append(*LINK(result));
          }
          else
              results.replace(*LINK(result), id);
      }
      void appendResult(IGraphResult * result)
      {
          CriticalBlock procedure(cs);
          results.append(*LINK(result));
      }
  };
  
  //===================================================================================================================
  
  class CPseudoArg : implements CThorSinkArgOf<IHThorArg>
  {
  };
  
  class CPseudoActivity : public CRoxieServerActivity
  {
  public:
      CPseudoActivity(IRoxieAgentContext *_ctx, IHThorArg & _helper) : CRoxieServerActivity(_ctx, _helper) {}
  
      virtual const void *nextRow()
      {
          throwUnexpected(); // I am nobody's input
      }
  };
  
  
  class CActivityGraph : implements IActivityGraph, implements IThorChildGraph, implements ILocalGraphEx, implements IRoxieServerChildGraph, public CInterface
  {
  protected:
      class ActivityGraphAgentContext : public IndirectAgentContext
      {
          SpinLock abortLock;
          bool aborted;
          Owned<IException> exception;
      public:
          ActivityGraphAgentContext(IRoxieAgentContext *_ctx, const IRoxieContextLogger &_logctx) : IndirectAgentContext(_ctx), logctx(_logctx), loopCounter(0), codeContext(NULL)
          {
              aborted = false;
          }
  
          // Note - we must track exceptions at the child level in case there is a CATCH in parent
  
          virtual void notifyAbort(IException *E)
          {
              if (QUERYINTERFACE(E, InterruptedSemaphoreException) == NULL)
              {
                  SpinBlock b(abortLock);
                  if (!aborted)
                  {
                      aborted = true;
                      exception.set(E);
                  }
              }
          }
  
          virtual void checkAbort()
          {
              if (aborted) // NOTE - don't bother getting lock before reading this (for speed) - a false read is very unlikely and not a problem
              {
                  SpinBlock b(abortLock);
                  if (!exception)
                      exception.setown(MakeStringException(ROXIE_INTERNAL_ERROR, "Query was aborted"));
                  throw exception.getLink();
              }
              IndirectAgentContext::checkAbort();
          }
  
          virtual ICodeContext *queryCodeContext()
          {
              return codeContext;
          }
          void setCodeContext(ICodeContext * _codeContext)
          {
              codeContext = _codeContext;
          }
          void setLoopCounter(unsigned _loopCounter)
          {
              loopCounter = _loopCounter;
          }
          void setPrefix(const char *name)
          {
              prefix.set(name);
          }
          virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph)
          {
              childGraphs.setValue(id, childGraph);
          }
          virtual IActivityGraph * queryChildGraph(unsigned  id)
          {
              if (queryTraceLevel() > 10)
                  CTXLOG("resolveChildGraph %d", id);
              IActivityGraph *childGraph = childGraphs.getValue(id);
              assertex(childGraph);
              return childGraph;
          }
          // MORE should really redirect the other log context ones too (though mostly doesn't matter). Really should refactor to have a queryLogContext() method in IRoxieAgentContext I think
          virtual StringBuffer &getLogPrefix(StringBuffer &ret) const
          {
              IndirectAgentContext::getLogPrefix(ret);
              if (prefix)
                  ret.appendf(":%s", prefix.str());
              if (loopCounter)
                  ret.appendf("{%u}", loopCounter);
              return ret;
          }
      protected:
          const IRoxieContextLogger &logctx;
          unsigned loopCounter;
          StringAttr prefix;
          ICodeContext * codeContext;
          MapXToMyClass<unsigned, unsigned, IActivityGraph> childGraphs;
      } graphAgentContext;
  
      class ActivityGraphCodeContext : public IndirectCodeContext
      {
      public:
          virtual IEclGraphResults * resolveLocalQuery(__int64 activityId)
          {
              if ((unsigned) activityId == container->queryId())
                  return container;
              IActivityGraph * match = agentContext->queryChildGraph((unsigned) activityId);
              if (match)
                  return match->queryLocalGraph();
              return IndirectCodeContext::resolveLocalQuery(activityId);
          }
          virtual IThorChildGraph * resolveChildQuery(__int64 activityId, IHThorArg * colocal)
          {
              IActivityGraph * match = agentContext->queryChildGraph((unsigned) activityId);
              return LINK(match->queryChildGraph());
          }
          virtual unsigned getGraphLoopCounter() const
          {
              return container->queryLoopCounter();           // only called if value is valid
          }
          void setContainer(IRoxieAgentContext * _agentContext, CActivityGraph * _container)
          {
              agentContext = _agentContext;
              container = _container;
          }
  
      protected:
          IRoxieAgentContext * agentContext;
          CActivityGraph * container;
      } graphCodeContext;
  
      class SinkThread : public CInterface, implements IThreaded
      {
      public:
          SinkThread(IActivityGraph &_parent, IRoxieServerActivity &_sink)
          : thread("SinkThread", this), parent(_parent), sink(_sink)
          {}
          virtual void threadmain() override
          {
              try
              {
                  sink.execute(parentExtractSize, parentExtract);
              }
              catch (IException *E)
              {
                  parent.noteException(E);
                  throw;
              }
          }
          void start(unsigned _parentExtractSize, const byte * _parentExtract)
          {
              parentExtract = _parentExtract;
              parentExtractSize = _parentExtractSize;
              thread.start();
          }
          inline void join()
          {
              thread.join(INFINITE);
          }
      private:
          CThreadedPersistent thread;
          IActivityGraph &parent;
          IRoxieServerActivity &sink;
          const byte * parentExtract = nullptr;
          unsigned parentExtractSize = 0;
      };
  
      // NOTE - destructor order is significant - need to destroy graphCodeContext and graphAgentContext last
  
      IArrayOf<IRoxieServerActivity> activities;
      IArrayOf<IRoxieProbe> probes;
      CIArrayOf<SinkThread> threads;
      IRoxieServerActivityCopyArray sinks;
      StringAttr graphName;
      Owned<CGraphResults> results;
      CGraphResults graphLoopResults;
      const ActivityArray & graphDefinition;
      CriticalSection evaluateCrit;
  
      IProbeManager *probeManager;
      IRoxieServerActivity *parentActivity;
      unsigned id;
      unsigned loopCounter;
      SinkMode sinkMode = SinkMode::Parallel;
  
  public:
      IMPLEMENT_IINTERFACE;
  
      CActivityGraph(IRoxieAgentContext *_ctx, const char *_graphName, unsigned _id, IRoxieServerActivity *_parentActivity, const ActivityArray &x, IProbeManager *_probeManager, const IRoxieContextLogger &_logctx)
          : graphAgentContext(_ctx, _logctx), graphName(_graphName), graphDefinition(x), probeManager(_probeManager), parentActivity(_parentActivity)
      {
          id = x.getLibraryGraphId();
          if (!id)
              id = _id;
          loopCounter = 0;
          graphAgentContext.setCodeContext(&graphCodeContext);
          graphCodeContext.setContainer(&graphAgentContext, this);
          if (parentActivity)
              sinkMode = parentActivity->queryFactory()->getSinkMode();
          else
              sinkMode = _ctx->queryOptions().sinkMode;
      }
  
      ~CActivityGraph()
      {
          if (probeManager)
              probeManager->deleteGraph((IArrayOf<IActivityBase>*)&activities, (IArrayOf<IInputBase>*)&probes);
      }
  
      virtual const char *queryName() const
      {
          return graphName.get();
      }
  
      virtual void setPrefix(const char *name) override
      {
          graphAgentContext.setPrefix(name);
      }
  
      void createGraph(IRoxieAgentContext *_ctx)
      {
          if (graphDefinition.isMultiInstance())
          {
              graphCodeContext.set(_ctx->queryCodeContext());
              _ctx = &graphAgentContext;
          }
  
          ForEachItemIn(idx, graphDefinition)
          {
              IRoxieServerActivityFactory &donor = graphDefinition.serverItem(idx);
              IRoxieServerActivity &activity = *donor.createActivity(_ctx, probeManager);
              activities.append(activity);
              if (donor.isSink())
              {
                  sinks.append(activity);
                  if (probeManager)
                      probeManager->noteSink(&activity);
              }
          }
          ForEachItemIn(idx1, graphDefinition)
          {
              IRoxieServerActivityFactory &donor = graphDefinition.serverItem(idx1);
              IRoxieServerActivity &activity = activities.item(idx1);
              unsigned inputidx = 0;
              for (;;)
              {
                  unsigned outputidx;
                  unsigned source = donor.getInput(inputidx, outputidx);
                  if (source==(unsigned) -1)
                      break;
  
                  connectInput(idx1, inputidx, source, outputidx, 0);
                  inputidx++;
              }
              IntArray &dependencies = donor.queryDependencies();
              IntArray &dependencyIndexes = donor.queryDependencyIndexes();
              IntArray &dependencyControlIds = donor.queryDependencyControlIds();
              StringArray &dependencyEdgeIds = donor.queryDependencyEdgeIds();
              ForEachItemIn(idx2, dependencies)
              {
                  IRoxieServerActivity &dependencySourceActivity = activities.item(dependencies.item(idx2));
                  unsigned dependencySourceIndex = dependencyIndexes.item(idx2);
                  unsigned dependencyControlId = dependencyControlIds.item(idx2);
                  activity.addDependency(dependencySourceActivity, dependencySourceIndex, dependencyControlId);
                  if (probeManager)
                      probeManager->noteDependency( &dependencySourceActivity, dependencySourceIndex, dependencyControlId, dependencyEdgeIds.item(idx2), &activity);
              }
          }
          ForEachItemIn(idx2, sinks)
          {
              IRoxieServerActivity &sink = sinks.item(idx2);
              sink.connectInputStreams(true);
          }
      }
  
      void connectInput(unsigned target, unsigned targetIdx, unsigned source, unsigned sourceIdx, unsigned iteration)
      {
          IRoxieServerActivity &targetActivity = activities.item(target);
          IRoxieServerActivity &sourceActivity = activities.item(source);
          IFinalRoxieInput * output = sourceActivity.queryOutput(sourceIdx);
          if (probeManager)
          {
              IRoxieProbe * inputProbe = probeManager->createProbe(static_cast<IInputBase*>(output), &sourceActivity, &targetActivity, sourceIdx, targetIdx, iteration);
              probes.append(*LINK(inputProbe));
              output = &dynamic_cast<IFinalRoxieInput &>(inputProbe->queryInput());
          }
          targetActivity.setInput(targetIdx, sourceIdx, output);
      }
  
      virtual void onCreate(IHThorArg *_colocalParent)
      {
          ForEachItemIn(idx, activities)
          {
              IRoxieServerActivity *activity = &activities.item(idx);
              if (activity)
                  activity->onCreate(_colocalParent);
          }
      }
  
      virtual void abort()
      {
          ForEachItemIn(idx, sinks)
          {
              IRoxieServerActivity &sink = sinks.item(idx);
              sink.abort();
          }
      }
  
      virtual void reset()
      {
          ForEachItemIn(idx, sinks)
          {
              IRoxieServerActivity &sink = sinks.item(idx);
              sink.reset();
          }
      }
  
      Linked<IException> exception;
      CriticalSection eCrit;
  
      virtual void noteException(IException *E)
      {
          CriticalBlock b(eCrit);
          if (!exception)
          {
              if (graphAgentContext.queryDebugContext())
              {
                  graphAgentContext.queryDebugContext()->checkBreakpoint(DebugStateException, NULL, exception);
              }
              exception.set(E);
          }
      }
  
      virtual void checkAbort() 
      {
          CriticalBlock b(eCrit);
          if (exception)
              throw exception.getLink();
      }
      unsigned queryWorkflowId() const
      {
          return graphDefinition.queryWorkflowId();
      }
      virtual void execute()
      {
          results.setown(new CGraphResults);
          doExecute(0, NULL);
      }
  
      //New child query code...
      virtual IThorChildGraph * queryChildGraph()
      {
          return this;
      }
  
      virtual IEclGraphResults * queryLocalGraph()
      {
          return this;
      }
  
      virtual IRoxieServerChildGraph * queryLoopGraph()
      {
          return this;
      }
  
      inline unsigned queryId() const
      {
          return id;
      }
  
      inline unsigned queryLoopCounter() const
      {
          return loopCounter;
      }
  
      void doExecute(unsigned parentExtractSize, const byte * parentExtract)
      {
          if (sinks.ordinality()==1)
              sinks.item(0).execute(parentExtractSize, parentExtract);
  #ifdef PARALLEL_EXECUTE
          else if (!probeManager && !graphDefinition.isSequential() && sinkMode != SinkMode::Sequential)
          {
              if (sinkMode == SinkMode::ParallelPersistent)
              {
                  if (!threads.ordinality())
                  {
                      for (unsigned i = 0; i < sinks.ordinality()-1; i++)
                      {
                          threads.append(*new SinkThread(*this, sinks.item(i)));
                      }
                  }
                  for (unsigned i = 0; i < sinks.ordinality()-1; i++)
                      threads.item(i).start(parentExtractSize, parentExtract);
                  try
                  {
                      sinks.item(sinks.ordinality()-1).execute(parentExtractSize, parentExtract);
                  }
                  catch (IException *E)
                  {
                      noteException(E);
                      E->Release();
                  }
                  for (unsigned i = 0; i < sinks.ordinality()-1; i++)
                  {
                      try
                      {
                          threads.item(i).join();
                      }
                      catch (IException *E)
                      {
                          noteException(E);
                          E->Release();
                      }
                  }
                  checkAbort();
              }
              else if (sinkMode == SinkMode::Parallel)
              {
                  class casyncfor: public CAsyncFor
                  {
                  public:
                      IActivityGraph &parent;
                      unsigned parentExtractSize;
                      const byte * parentExtract;
  
                      casyncfor(IRoxieServerActivityCopyArray &_sinks, IActivityGraph &_parent, unsigned _parentExtractSize, const byte * _parentExtract) :
                          parent(_parent), parentExtractSize(_parentExtractSize), parentExtract(_parentExtract), sinks(_sinks) { }
                      void Do(unsigned i)
                      {
                          try
                          {
                              sinks.item(i).execute(parentExtractSize, parentExtract);
                          }
                          catch (IException *E)
                          {
                              parent.noteException(E);
                              throw;
                          }
                      }
                  private:
                      IRoxieServerActivityCopyArray &sinks;
                  } afor(sinks, *this, parentExtractSize, parentExtract);
                  afor.For(sinks.ordinality(), sinks.ordinality());
              }
              else
                  throwUnexpected();
          }
  #endif
          else
          {
              ForEachItemIn(idx, sinks)
              {
                  IRoxieServerActivity &sink = sinks.item(idx);
                  sink.execute(parentExtractSize, parentExtract);
              }
          }
      }
  
      virtual IEclGraphResults *evaluate(unsigned parentExtractSize, const byte * parentExtract)
      {
          CriticalBlock block(evaluateCrit);
          results.setown(new CGraphResults);
          try
          {
              doExecute(parentExtractSize, parentExtract);
          }
          catch (...)
          {
              DBGLOG("Exception thrown in child query - cleaning up");
              reset();
              throw;
          }
          reset();
          return results.getClear();
      }
  
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override
      {
          if (id && statsBuilder)
          {
              statsBuilder->beginSubGraphScope(parentActivity->querySubgraphId());
              statsBuilder->beginActivityScope(parentActivity->queryId());
              statsBuilder->beginChildGraphScope(id);
          }
  
          ForEachItemIn(i, activities)
              activities.item(i).gatherStatistics(statsBuilder);
  
          if (id && statsBuilder)
          {
              statsBuilder->endScope();
              statsBuilder->endScope();
              statsBuilder->endScope();
          }
      }
  
      //interface IRoxieServerChildGraph
      virtual void beforeExecute()
      {
          results.setown(new CGraphResults);
      }
  
      virtual IFinalRoxieInput * startOutput(unsigned id, unsigned parentExtractSize, const byte *parentExtract, bool paused)
      {
          IFinalRoxieInput * ret = selectOutput(id);
          ret->start(parentExtractSize, parentExtract, paused);
          return ret;
      }
  
      virtual IFinalRoxieInput * selectOutput(unsigned id)
      {
          ForEachItemIn(i, sinks)
          {
              IFinalRoxieInput * ret = sinks.item(i).querySelectOutput(id);
              if (ret)
                  return ret;
          }
          throwUnexpected();
          return NULL;
      }
  
      virtual void setInputResult(unsigned id, IGraphResult * result)
      {
          results->setResult(id, result);
      }
  
      virtual bool querySetInputResult(unsigned id, unsigned _sourceIdx, IFinalRoxieInput * input)
      {
          ForEachItemIn(i, activities)
          {
              if (activities.item(i).querySetStreamInput(id, _sourceIdx, input))
                  return true;
          }
          return false;
      }
  
      virtual void stopUnusedOutputs()
      {
          //Hmm not sure how to do this...
      }
  
      virtual void afterExecute()
      {
          ForEachItemIn(i, sinks)
          {
              sinks.item(i).stop();
          }
          if (graphAgentContext.queryDebugContext())
          {
              graphAgentContext.queryDebugContext()->checkBreakpoint(DebugStateGraphFinished, NULL, NULL);
          }
          reset();
      }
  
      virtual IRoxieGraphResults * execute(size32_t parentExtractSize, const byte *parentExtract)
      {
          doExecute(parentExtractSize, parentExtract);
          return LINK(results);
      }
      virtual void getLinkedResult(unsigned & count, const byte * * & ret, unsigned id) override
      {
          results->getLinkedResult(count, ret, id);
      }
      virtual void getDictionaryResult(unsigned & count, const byte * * & ret, unsigned id) override
      {
          results->getLinkedResult(count, ret, id);
      }
      virtual const void * getLinkedRowResult(unsigned id)
      {
          return results->getLinkedRowResult(id);
      }
      virtual void setResult(unsigned id, IGraphResult * result)
      {
          results->setResult(id, result);
      }
      virtual IEngineRowStream * createResultIterator(unsigned id)
      {
          return results->createIterator(id);
      }
      virtual void setGraphLoopResult(IGraphResult * result)
      {
          graphLoopResults.appendResult(result);
      }
      virtual IEngineRowStream * createGraphLoopResultIterator(unsigned id)
      {
          try
          {
              return graphLoopResults.createIterator(id);
          }
          catch (IException * e)
          {
              e->Release();
              throw MakeStringException(ROXIE_GRAPH_PROCESSING_ERROR, "Error reading graph result %d before it is calculated", id); 
          }
      }
      virtual void clearGraphLoopResults()
      {
          graphLoopResults.clear();
      }
      virtual void executeGraphLoop(size32_t parentExtractSize, const byte *parentExtract)
      {
          doExecute(parentExtractSize, parentExtract);
      }
      virtual void setGraphLoopResult(unsigned id, IGraphResult * result)
      {
          graphLoopResults.setResult(id, result);
      }
      virtual IEngineRowStream * getGraphLoopResult(unsigned id)
      {
          return graphLoopResults.createIterator(id);
      }
  
      virtual void getProbeResponse(IPropertyTree *query)
      {
          if (probeManager)
              probeManager->getProbeResponse(query);
      }
  
      virtual IRoxieServerChildGraph * createGraphLoopInstance(IRoxieAgentContext *ctx, unsigned loopCounter, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &logctx)
      {
          throwUnexpected();
      }
  
      virtual CGraphIterationInfo *selectGraphLoopOutput()
      {
          return NULL;
      }
  
      virtual void gatherIterationUsage(IRoxieServerLoopResultProcessor & processor)
      {
          throwUnexpected();
      }
  
      virtual void associateIterationOutputs(IRoxieServerLoopResultProcessor & processor)
      {
          throwUnexpected();
      }
  };
  
  class CProxyActivityGraph : implements IActivityGraph, implements IThorChildGraph, public CInterface
  {
  public:
      IMPLEMENT_IINTERFACE;
  
      CProxyActivityGraph(IRoxieAgentContext *_ctx, const char *_graphName, unsigned _id, IRoxieServerActivity *_parentActivity, ActivityArray &_graphDefinition, const IRoxieContextLogger &_logctx, unsigned _numParallel)
          : ctx(_ctx), parentActivity(_parentActivity), graphName(_graphName), id(_id), graphDefinition(_graphDefinition), logctx(_logctx), numParallel(_numParallel)
      {
      }
      virtual void abort() override { throwUnexpected(); }
      virtual void reset() override { throwUnexpected(); }
      virtual void execute() override { throwUnexpected(); }
      virtual void getProbeResponse(IPropertyTree *query) override { throwUnexpected(); }
      virtual void onCreate(IHThorArg *_colocalArg) override
      {
          colocalArg = _colocalArg;
      }
      virtual void noteException(IException *E) override { throwUnexpected(); }
      virtual void checkAbort() override { throwUnexpected(); }
      virtual IThorChildGraph * queryChildGraph() override { return this; }
      virtual IEclGraphResults * queryLocalGraph() override { throwUnexpected(); }
      virtual IRoxieServerChildGraph * queryLoopGraph() override { throwUnexpected(); }
      virtual IRoxieServerChildGraph * createGraphLoopInstance(IRoxieAgentContext *ctx, unsigned loopCounter, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &logctx) override { throwUnexpected(); }
      virtual const char *queryName() const override { throwUnexpected(); }
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override
      {
          CriticalBlock b(graphCrit);
          ForEachItemIn(i, stack)
              stack.item(i).gatherStatistics(statsBuilder);
      }
      virtual IEclGraphResults * evaluate(unsigned parentExtractSize, const byte * parentExtract) override
      {
          Owned<CActivityGraph> realGraph;
          {
              CriticalBlock b(graphCrit);
              if (stack.length())
                  realGraph.setown(&stack.popGet());
          }
          if (!realGraph)
          {
              realGraph.setown(new CActivityGraph(ctx, graphName, id, parentActivity, graphDefinition, NULL, logctx));
              realGraph->createGraph(ctx);
              realGraph->onCreate(colocalArg);
          }
          Owned<IEclGraphResults> results = realGraph->evaluate(parentExtractSize, parentExtract);
          {
              CriticalBlock b(graphCrit);
              stack.append(*realGraph.getClear());
          }
          return results.getClear();
      }
      virtual void setPrefix(const char *) override
      {
      }
      virtual unsigned queryWorkflowId() const override
      {
          return graphDefinition.queryWorkflowId();
      }
  protected:
      IRoxieAgentContext *ctx;
      IRoxieServerActivity *parentActivity;
      StringAttr graphName;
      unsigned id;
      ActivityArray &graphDefinition;
      const IRoxieContextLogger &logctx;
      unsigned numParallel;
      IHThorArg *colocalArg = nullptr;
      mutable CriticalSection graphCrit;
      CIArrayOf<CActivityGraph> stack;
  };
  
  class CIterationActivityGraph : public CActivityGraph
  {
      IHThorArg * colocalParent;
      unsigned fixedParentExtractSize;
      const byte * fixedParentExtract;
      unsigned graphOutputActivityIndex;
  
  public:
      CIterationActivityGraph(const char *_graphName, unsigned _id, IRoxieServerActivity *_parentActivity, ActivityArray &x, IProbeManager *_probeManager,
                              unsigned _loopCounter, IRoxieAgentContext *_ctx, IHThorArg * _colocalParent, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &_logctx)
          : CActivityGraph(_ctx, _graphName, _id, _parentActivity, x, _probeManager, _logctx)
      {
          graphOutputActivityIndex = 0;
          loopCounter = _loopCounter;
          colocalParent = _colocalParent;
          graphAgentContext.setLoopCounter(loopCounter);
          graphCodeContext.set(_ctx->queryCodeContext());
  
          fixedParentExtractSize = parentExtractSize;
          fixedParentExtract = parentExtract;
      }
  
      void createIterationGraph(IRoxieAgentContext *_ctx)
      {
          Owned<IRoxieServerActivity> pseudoActivity = new CPseudoActivity(_ctx, *new CPseudoArg);
  
          ForEachItemIn(idx1, graphDefinition)
              activities.append(*LINK(pseudoActivity));
  
          graphOutputActivityIndex = queryGraphOutputIndex();
          recursiveCreateGraph(_ctx, graphOutputActivityIndex);
      }
  
      unsigned queryGraphOutputIndex() const
      {
          ForEachItemIn(i, graphDefinition)
              if (graphDefinition.serverItem(i).getKind() == TAKgraphloopresultwrite)
                  return i;
          throwUnexpected();
      }
  
      void recursiveCreateGraph(IRoxieAgentContext *_ctx, unsigned whichActivity)
      {
          //Check to see if already created
          IRoxieServerActivity & prevActivity = activities.item(whichActivity);
          if (prevActivity.queryId() != 0)
          {
              prevActivity.noteOutputUsed();      //We need to patch up the number of outputs for splitters.
              return;
          }
  
          IRoxieServerActivityFactory &donor = graphDefinition.serverItem(whichActivity);
          IRoxieServerActivity * activity = NULL;
          if (donor.isGraphInvariant())
          {
              ThorActivityKind kind = donor.getKind();
              switch (kind)
              {
              case TAKif:
              case TAKchildif:
              case TAKcase:
              case TAKchildcase:  // MORE RKC->GH - what about FILTER with a graph-invariant condition and other latestart activities?
                  {
                      Owned<IHThorArg> helper = &donor.getHelper();
                      helper->onCreate(&graphCodeContext, colocalParent, NULL);
                      helper->onStart(fixedParentExtract, NULL);
  
                      unsigned branch;
                      switch (kind)
                      {
                      case TAKif: case TAKchildif:
                          branch = static_cast<IHThorIfArg *>(helper.get())->getCondition() ? 0 : 1;
                          break;
                      case TAKcase: case TAKchildcase:
                          branch = static_cast<IHThorCaseArg *>(helper.get())->getBranch();
                          if (branch >= donor.numInputs())
                              branch = donor.numInputs() - 1;
                          break;
                      default:
                          throwUnexpected();
                      }
                      helper.clear();
  
                      unsigned outputidx;
                      unsigned source = donor.getInput(branch, outputidx);
                      if (source ==(unsigned) -1)
                          activity = createRoxieServerNullActivity(&graphAgentContext, &donor, probeManager);
                      else
                          activity = createRoxieServerPassThroughActivity(&graphAgentContext, &donor, probeManager);
  
                      activities.replace(*activity, whichActivity);
                      activity->onCreate(colocalParent);
                      if (source ==(unsigned) -1)
                          return;
  
                      recursiveCreateGraph(_ctx, source);
                      connectInput(whichActivity, 0, source, outputidx, loopCounter);
                      break;
                  }
              }
          }
  
          if (!activity)
          {
              activity = donor.createActivity(&graphAgentContext, probeManager);
              activities.replace(*activity, whichActivity);
              activity->onCreate(colocalParent);
              activity->resetOutputsUsed();
  
              unsigned inputidx = 0;
              for (;;)
              {
                  unsigned outputidx;
                  unsigned source = donor.getInput(inputidx, outputidx);
                  if (source==(unsigned) -1)
                      break;
  
                  recursiveCreateGraph(_ctx, source);
                  connectInput(whichActivity, inputidx, source, outputidx, loopCounter);
                  inputidx++;
              }
          }
  
          IntArray &dependencies = donor.queryDependencies();
          IntArray &dependencyIndexes = donor.queryDependencyIndexes();
          IntArray &dependencyControlIds = donor.queryDependencyControlIds();
          ForEachItemIn(idx2, dependencies)
          {
              unsigned input = dependencies.item(idx2);
              recursiveCreateGraph(_ctx, input);
              activity->addDependency(activities.item(input),dependencyIndexes.item(idx2),dependencyControlIds.item(idx2));
          }
      }
  
      virtual CGraphIterationInfo *selectGraphLoopOutput()
      {
          IRoxieServerActivity &sourceActivity = activities.item(graphOutputActivityIndex);
          IFinalRoxieInput *sourceInput = sourceActivity.queryOutput(0);
          return new CGraphIterationInfo(&sourceActivity, sourceInput, 0, loopCounter);
      }
  
      virtual void gatherIterationUsage(IRoxieServerLoopResultProcessor & processor)
      {
          ForEachItemIn(i, activities)
              activities.item(i).gatherIterationUsage(processor, fixedParentExtractSize, fixedParentExtract);
      }
  
      virtual void associateIterationOutputs(IRoxieServerLoopResultProcessor & processor)
      {
          ForEachItemIn(i, activities)
              activities.item(i).associateIterationOutputs(processor, fixedParentExtractSize, fixedParentExtract, probeManager, probes);
      }
  
  };
  
  
  class CDelayedActivityGraph : implements IActivityGraph, public CInterface
  {
      StringAttr graphName;
      StringAttr prefix;
      ActivityArray & graphDefinition;
      IProbeManager *probeManager;
      unsigned id;
      IRoxieAgentContext * ctx;
      IHThorArg * colocalParent;
      IRoxieServerActivity *parentActivity;
  
  public:
      IMPLEMENT_IINTERFACE;
  
      CDelayedActivityGraph(const char *_graphName, unsigned _id, IRoxieServerActivity *_parentActivity, ActivityArray &x, IProbeManager *_probeManager)
          : graphDefinition(x), probeManager(_probeManager), parentActivity(_parentActivity)
      {
          graphName.set(_graphName);
          id = _id;
          ctx = NULL;
          colocalParent = NULL;
      }
  
      virtual const char *queryName() const override { return graphName.get(); }
      virtual void abort() override { throwUnexpected(); }
      virtual void reset() override { }
      virtual void execute() override { throwUnexpected(); }
      virtual void getProbeResponse(IPropertyTree *query) override { throwUnexpected(); }
      virtual void noteException(IException *E) override { throwUnexpected(); }
      virtual void checkAbort() override { throwUnexpected(); }
      virtual IThorChildGraph * queryChildGraph() override { throwUnexpected(); }
      virtual IEclGraphResults * queryLocalGraph() override { throwUnexpected(); }
      virtual IRoxieServerChildGraph * queryLoopGraph() override { throwUnexpected(); }
      virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const override { }
      virtual void onCreate(IHThorArg *_colocalParent) override
      { 
          colocalParent = _colocalParent;
      }
      virtual void setPrefix(const char *pfx) override
      {
          prefix.set(pfx);
      }
      virtual unsigned queryWorkflowId() const override
      {
          return graphDefinition.queryWorkflowId();
      }
      virtual IRoxieServerChildGraph * createGraphLoopInstance(IRoxieAgentContext *ctx, unsigned loopCounter, unsigned parentExtractSize, const byte * parentExtract, const IRoxieContextLogger &logctx) override
      {
          Owned<CIterationActivityGraph> ret = new CIterationActivityGraph(graphName, id, parentActivity, graphDefinition, probeManager, loopCounter, ctx, colocalParent, parentExtractSize, parentExtract, logctx);
          ret->createIterationGraph(ctx);
          if (prefix)
              ret->setPrefix(prefix);
          return ret.getClear();
      }
  };
  
  
  
  IActivityGraph *createActivityGraph(IRoxieAgentContext *ctx, const char *_graphName, unsigned id, ActivityArray &childFactories, IRoxieServerActivity *parentActivity, IProbeManager *_probeManager, const IRoxieContextLogger &_logctx, unsigned numParallel)
  {
      if (childFactories.isDelayed())
      {
          assertex(numParallel==1);
          return new CDelayedActivityGraph(_graphName, id, parentActivity, childFactories, _probeManager);
      }
      else
      {
          if (numParallel==1 || _probeManager != nullptr)
          {
              Owned<IProbeManager> childProbe;
              if (_probeManager)
                  childProbe.setown(_probeManager->startChildGraph(id, parentActivity));
              Owned<CActivityGraph> ret = new CActivityGraph(ctx, _graphName, id, parentActivity, childFactories, childProbe, _logctx);
              ret->createGraph(ctx);
              if (_probeManager)
                  _probeManager->endChildGraph(childProbe, parentActivity);
              return ret.getClear();
          }
          else
              return new CProxyActivityGraph(ctx, _graphName, id, parentActivity, childFactories, _logctx, numParallel);
      }
  }
  
  //================================================================================================================
  
  #ifdef _USE_CPPUNIT
  #include "unittests.hpp"
  
  // There is a bug in VC6 implemetation of protected which prevents nested classes from accessing owner's data. It can be tricky to work around - hence...
  #if _MSC_VER==1200
  #undef protected
  #endif
  
  static const char *sortAlgorithm;
  
  class TestMetaData : public CInterface, implements IOutputMetaData 
  {
  public:
      IMPLEMENT_IINTERFACE;
      virtual size32_t getRecordSize(const void *) { return 10; }
      virtual size32_t getMinRecordSize() const { return 10; }
      virtual size32_t getFixedSize() const { return 10; }
      virtual void toXML(const byte * self, IXmlWriter & out) {}
      virtual unsigned getVersion() const                     { return OUTPUTMETADATA_VERSION; }
      virtual unsigned getMetaFlags()                         { return 0; }
      virtual const RtlTypeInfo * queryTypeInfo() const { return nullptr; }
      virtual void destruct(byte * self)  {}
      virtual IOutputRowSerializer * createDiskSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual IOutputRowDeserializer * createDiskDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual ISourceRowPrefetcher * createDiskPrefetcher() { return NULL; }
      virtual IOutputMetaData * querySerializedDiskMeta() { return NULL; }
      virtual IOutputRowSerializer * createInternalSerializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual IOutputRowDeserializer * createInternalDeserializer(ICodeContext * ctx, unsigned activityId) { return NULL; }
      virtual void process(const byte * self, IFieldProcessor & target, unsigned from, unsigned to) {}
      virtual void walkIndirectMembers(const byte * self, IIndirectMemberVisitor & visitor) {}
      virtual IOutputMetaData * queryChildMeta(unsigned i) { return NULL; }
      virtual const RtlRecord &queryRecordAccessor(bool expand) const { throwUnexpected(); }
  } testMeta;
  
  class TestInput : public CInterface, implements IFinalRoxieInput, implements IEngineRowStream
  {
      char const * const *input;
      IRoxieAgentContext *ctx;
      unsigned endSeen;
      bool eof;
      unsigned count;
      ActivityTimeAccumulator activityStats;
      size32_t recordSize;
      unsigned activityId;
  
  public:
      enum { STATEreset, STATEstarted, STATEstopped } state;
      bool allRead;
      unsigned repeat;
      IMPLEMENT_IINTERFACE;
      TestInput(IRoxieAgentContext *_ctx, char const * const *_input) 
      { 
          ctx = _ctx; 
          input = _input; 
          count = 0;
          eof = false;
          allRead = false;
          endSeen = 0;
          recordSize = testMeta.getFixedSize();
          state = STATEreset;
          activityId = 1;
          repeat = 0;
      }
      virtual IOutputMetaData * queryOutputMeta() const { return &testMeta; }
      virtual void prestart(unsigned parentExtractSize, const byte *parentExtract)
      {
          ASSERT(state == STATEreset);
      }
      virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused) 
      {
          ASSERT(state == STATEreset);
          state = STATEstarted; 
      }
      virtual IStrandJunction *getOutputStreams(IRoxieAgentContext *ctx, unsigned idx, PointerArrayOf<IEngineRowStream> &streams, const StrandOptions * consumerOptions, bool consumerOrdered, IOrderedCallbackCollection * orderedCallbacks)
      {
          streams.append(this);
          return NULL;
      }
      virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
      virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
      virtual IRoxieServerActivity *queryActivity()
      {
          throwUnexpected();
      }
      virtual IIndexReadActivityInfo *queryIndexReadActivity()
      {
          throwUnexpected();
      }
      virtual void stop()
      {
          state = STATEstopped; 
      }
      virtual void reset() 
      {
          ASSERT(state == STATEstopped);
          eof = false; count = 0; endSeen = 0; allRead = false; state = STATEreset;
      }
      virtual void resetEOF()
      {
          throwUnexpected();
      }
      virtual void checkAbort() {}
      virtual unsigned queryId() const { return activityId; };
      virtual const void *nextRow() 
      {
          ActivityTimer t(activityStats, ctx->queryOptions().timeActivities);
          ASSERT(state == STATEstarted);
          ASSERT(allRead || !eof);
          if (eof)
              return NULL;
      again:
          const char *nextSource = input[count++];
          if (nextSource)
          {
              endSeen = 0;
              void *ret = ctx->queryRowManager().ALLOCATE(recordSize);
              memset(ret, 0, recordSize);
              strncpy((char *) ret, nextSource, recordSize);
              return ret;
          }
          else if (repeat)
          {
              repeat--;
              count = 0;
              goto again;
          }
          else
          {
              endSeen++;
              if (endSeen==2)
                  eof = true;
              return NULL;
          }
      }
      virtual bool nextGroup(ConstPointerArray & group) 
      {
          const void * next;
          while ((next = nextRow()) != NULL)
              group.append(next);
          if (group.ordinality())
              return true;
          return false;
      }
      virtual unsigned __int64 queryTotalCycles() const { return activityStats.totalCycles; }
      virtual unsigned __int64 queryLocalCycles() const { return activityStats.totalCycles; }
      virtual IFinalRoxieInput *queryInput(unsigned idx) const
      {
          return NULL;
      }
  
  };
  
  struct SortActivityTest : public CThorSortArg {
  public:
      struct CompareClass : public ICompare {
          virtual int docompare(const void * _left, const void * _right) const {
              return memcmp(_left, _right, 5);
          }
      } compare;
      virtual ICompare * queryCompare() { return &compare; }
      virtual IOutputMetaData * queryOutputMeta() 
      { 
          return &testMeta; 
      }
      virtual unsigned getAlgorithmFlags() { return TAFunstable; }
      virtual const char * getAlgorithm() { return sortAlgorithm; }
  };
  extern "C" IHThorArg * sortActivityTestFactory() { return new SortActivityTest; }
  
  struct MergeActivityTest : public CThorMergeArg {
      static bool isDedup;
  public:
      struct CompareClass : public ICompare {
          virtual int docompare(const void * _left, const void * _right) const {
              return memcmp(_left, _right, 5);
          }
      } compare;
      virtual ICompare * queryCompare() { return &compare; }
      virtual IOutputMetaData * queryOutputMeta() 
      { 
          return &testMeta; 
      }
      virtual bool dedup() { return isDedup; }
  };
  bool MergeActivityTest::isDedup = false;
  extern "C" IHThorArg * mergeActivityTestFactory() { return new MergeActivityTest; }
  
  struct SplitActivityTest : public CThorSplitArg {
  public:
      virtual unsigned numBranches() { return 2; }
      virtual IOutputMetaData * queryOutputMeta()
      {
          return &testMeta;
      }
  };
  extern "C" IHThorArg * splitActivityTestFactory() { return new SplitActivityTest; }
  
  class CcdServerTest : public CppUnit::TestFixture  
  {
      CPPUNIT_TEST_SUITE(CcdServerTest);
          CPPUNIT_TEST(testSetup);
          CPPUNIT_TEST(testHeapSort);
          CPPUNIT_TEST(testQuickSort);
          CPPUNIT_TEST(testMerge);
          CPPUNIT_TEST(testMergeDedup);
          CPPUNIT_TEST(testMiscellaneous);
          CPPUNIT_TEST(testSplitter);
          CPPUNIT_TEST(testCleanup);
      CPPUNIT_TEST_SUITE_END();
  protected:
      AgentContextLogger logctx;
      Owned<const IQueryDll> queryDll;
      Owned<IRoxiePackage> package;
      Owned<IPropertyTree> stateInfo;
      Owned<IQueryFactory> queryFactory;
      Owned<IRoxieAgentContext> ctx;
  
      void testSetup()
      {
          selfTestMode = true;
          roxiemem::setTotalMemoryLimit(false, true, false, 100 * 1024 * 1024, 0, NULL, NULL);
      }
  
      void testCleanup()
      {
          roxiemem::releaseRoxieHeap();
      }
  
      void init()
      {
          package.setown(createRoxiePackage(NULL, NULL));
          queryDll.setown(createExeQueryDll("roxie"));
          stateInfo.setown(createPTreeFromXMLString("<test memoryLimit='50000000'/>"));
          queryFactory.setown(createServerQueryFactory("test", queryDll.getLink(), *package, stateInfo, false, false));
          ctx.setown(createAgentContext(queryFactory, logctx, NULL, false));
          queryActiveTimer()->reset();
      }
  
      void testActivity(IRoxieServerActivity *activity, char const * const *input, char const * const *output)
      {
          testActivity(activity, input, NULL, output);
      }
  
      void testActivity(IRoxieServerActivity *activity, char const * const *input, char const * const *input2, char const * const *output, unsigned outputIdx = 0)
      {
          TestInput in(ctx, input);
          TestInput in2(ctx, input2);
          activity->setInput(0, 0, &in);
          if (input2)
              activity->setInput(1, 0, &in2);
          IFinalRoxieInput *out = activity->queryOutput(outputIdx);
          IEngineRowStream *outStream = connectSingleStream(ctx, out, outputIdx, true);
          IOutputMetaData *meta = out->queryOutputMeta();
          void *buf = alloca(meta->getFixedSize());
  
          for (unsigned iteration = 0; iteration < 8; iteration++)
          {
              // All activities should be able to be restarted multiple times in the same context (for child queries) or in a new context (for graph pooling, if we ever wanted it)
              // This should be true whether we read all, some, or none of the data.
              // Should not matter if an activity is not started
              if (iteration % 4 == 0)
                  activity->onCreate(NULL);
              unsigned count = 0;
              if (iteration % 4 != 3)
              {
                  activity->start(0, NULL, false);
                  ASSERT(in.state == TestInput::STATEstarted);
                  ASSERT(!input2 || in2.state == TestInput::STATEstarted);
                  for (;;)
                  {
                      const void *next = outStream->nextRow();
                      if (!next)
                      {
                          ASSERT(output[count++] == NULL);
                          next = outStream->nextRow();
                          if (!next)
                          {
                              ASSERT(output[count++] == NULL);
                              break;
                          }
                      }
                      ASSERT(output[count] != NULL);
                      unsigned outsize = meta->getRecordSize(next);
                      memset(buf, 0, outsize);
                      strncpy((char *) buf, output[count++], outsize);
                      ASSERT(memcmp(next, buf, outsize) == 0);
                      ReleaseRoxieRow(next);
                      if (iteration % 4 == 2)
                          break;
                  }
                  if (iteration % 4 != 2)
                  {
                      // Check that reading after end is harmless
                      in.allRead = true;
                      const void *next = outStream->nextRow();
                      ASSERT(next == NULL);
                  }
              }
              activity->stop();
              ASSERT(in.state == TestInput::STATEstopped);
              ASSERT(!input2 || in2.state == TestInput::STATEstopped);
              activity->reset();
              ASSERT(in.state == TestInput::STATEreset);
              ASSERT(!input2 || in2.state == TestInput::STATEreset);
              ctx->queryRowManager().reportLeaks();
              ASSERT(ctx->queryRowManager().numPagesAfterCleanup(true) == 0);
          }
      }
  
      void testSplitActivity(IRoxieServerActivityFactory *factory, char const * const *input, char const * const *output, unsigned numOutputs, unsigned repeat)
      {
          Owned <IRoxieServerActivity> activity = factory->createActivity(ctx, NULL);
          TestInput in(ctx, input);
          in.repeat = repeat;
          activity->setInput(0, 0, &in);
          ArrayOf<IFinalRoxieInput *> out;
          for (unsigned i = 0; i < numOutputs; i++)
          {
              out.append(activity->queryOutput(i));
          }
          activity->onCreate(NULL);
  
          class casyncfor: public CAsyncFor
          {
          public:
              casyncfor(ArrayOf<IFinalRoxieInput *> &_outputs, char const * const *_output, unsigned _repeat)
              : outputs(_outputs), output(_output), repeat(_repeat)
              {}
              void Do(unsigned i)
              {
                  IFinalRoxieInput *out = outputs.item(i);
                  IEngineRowStream *outStream = connectSingleStream(NULL, out, 0, true);
                  out->start(0, NULL, false);
                  IOutputMetaData *meta = out->queryOutputMeta();
                  void *buf = alloca(meta->getFixedSize());
                  unsigned count = 0;
                  unsigned repeats = repeat;
                  for (;;)
                  {
                      const void *next = outStream->nextRow();
                      if (!next)
                      {
                          ASSERT(repeats==0);
                          ASSERT(output[count] == NULL);
                          count++;
                          next = outStream->nextRow();
                          if (!next)
                          {
                              ASSERT(output[count] == NULL);
                              count++;
                              break;
                          }
                      }
                      if (output[count] == NULL && repeats)
                      {
                          repeats--;
                          count = 0;
                      }
                      ASSERT(output[count] != NULL);
                      unsigned outsize = meta->getRecordSize(next);
                      memset(buf, 0, outsize);
                      strncpy((char *) buf, output[count++], outsize);
                      ASSERT(memcmp(next, buf, outsize) == 0);
                      ReleaseRoxieRow(next);
                  }
                  outStream->stop();
              }
          private:
              ArrayOf<IFinalRoxieInput *> &outputs;
              char const * const *output;
              unsigned repeat;
          } afor(out, output, repeat);
          afor.For(numOutputs, numOutputs);
  
          ASSERT(in.state == TestInput::STATEstopped);
          for (unsigned i2 = 0; i2 < numOutputs; i2++)
          {
              out.item(i2)->reset();
          }
          ASSERT(in.state == TestInput::STATEreset);
          ctx->queryRowManager().reportLeaks();
          ASSERT(ctx->queryRowManager().numPagesAfterCleanup(true) == 0);
      }
  
      static int compareFunc(const void *l, const void *r)
      {
          return strcmp(*(char **) l, *(char **) r);
      }
  
      void testSort(unsigned type)
      {
          init();
          sortAlgorithm = NULL;
          if (type==2)
              sortAlgorithm = "heapSort";
          else
              sortAlgorithm = "quickSort";
          DBGLOG("Testing %s activity", sortAlgorithm);
          Owned<IPropertyTree> dummyNode = createPTree("node");
          Owned <IRoxieServerActivityFactory> factory = createRoxieServerSortActivityFactory(1, 1, *queryFactory, sortActivityTestFactory, TAKsort, *dummyNode);
          Owned <IRoxieServerActivity> activity = factory->createActivity(ctx, NULL);
          const char * test[] = { NULL, NULL };
          const char * test12345[] = { "1", "2", "3", "4", "5", NULL, NULL };
          const char * test54321[] = { "5", "4", "3", "2", "1", NULL, NULL };
          const char * test11111[] = { "1", "1", "1", "1", "1", NULL, NULL };
          const char * test11111_12345[] = { "1", "1", "1", "1", "1", NULL, "1", "2", "3", "4", "5", NULL, NULL };
          const char * test11111_54321[] = { "1", "1", "1", "1", "1", NULL, "5", "4", "3", "2", "1", NULL, NULL };
          const char * test54321_54321[] = { "5", "4", "3", "2", "1", NULL, "5", "4", "3", "2", "1", NULL, NULL };
          const char * test12345_12345[] = { "1", "2", "3", "4", "5", NULL, "1", "2", "3", "4", "5", NULL, NULL };
          testActivity(activity, test, test);
          testActivity(activity, test12345, test12345);
          testActivity(activity, test54321, test12345);
          testActivity(activity, test11111, test11111);
          testActivity(activity, test11111_12345, test11111_12345);
          testActivity(activity, test11111_54321, test11111_12345);
          testActivity(activity, test54321_54321, test12345_12345);
  
          // A few larger tests
          char *input[2002];
          char *output[2002];
          input[2000] = input[2001] = output[2000] = output[2001] = NULL;
  
          unsigned i;
          // identical
          for (i=0; i<2000; i++)
          {
              input[i] = new char[11];
              output[i] = new char[11];
              sprintf(input[i], "1");
              sprintf(output[i], "1");
          }
          testActivity(activity, input, output);
          // Ascending
          for (i=0; i<2000; i++)
          {
              sprintf(input[i], "%04d", i);
              sprintf(output[i], "%04d", i);
          }
          testActivity(activity, input, output);
          // Almost sorted
          for (i=0; i<20; i++)
          {
              unsigned h = i*100;
              sprintf(input[h], "%04d", 1900-h);
          }
          testActivity(activity, input, output);
          // Descending
          for (i=0; i<2000; i++)
          {
              sprintf(input[i], "%04d", 1999-i);
              sprintf(output[i], "%04d", i);
          }
          testActivity(activity, input, output);
          // Random
          for (i=0; i<2000; i++)
          {
              unsigned r = fastRand() % 1500;
              sprintf(input[i], "%04d", r);
              sprintf(output[i], "%04d", r);
          }
          qsort(output, 2000, sizeof(output[0]), compareFunc);
          testActivity(activity, input, output);
  
          unsigned __int64 us = factory->queryLocalTimeNs();
          DBGLOG("Simple %s sorts: activity time %u.%u ms", type==2?"Heap" : (type==1 ? "Insertion" : "Quick"), (int)(us/1000), (int)(us%1000));
          factory->resetNodeProgressInfo();
          if (type)
          {
              // Other than quicksort, it's supposed to be stable. Let's check that it is
              // All sort identical
              for (i=0; i<2000; i++)
              {
                  sprintf(input[i], "1    %d", i);
                  sprintf(output[i], "1    %d", i);
              }
              testActivity(activity, input, output);
              // Already sorted
              for (i=0; i<2000; i++)
              {
                  sprintf(input[i], "%04d %d", i / 10, i);
                  sprintf(output[i], "%04d %d", i / 10, i);
              }
              testActivity(activity, input, output);
              // Reverse order
              for (i=0; i<2000; i++)
              {
                  sprintf(input[i], "%04d %d", 199 - (i / 10), i%10);
                  sprintf(output[i], "%04d %d", i / 10, i%10);
              }
              testActivity(activity, input, output);
          }
          for (i=0; i<2000; i++)
          {
              delete [] input[i];
              delete [] output[i];
          }
          DBGLOG("Finished testing %s sort", type==2?"Heap" : (type==1 ? "Insertion" : "Quick"));
      }
  
      void testQuickSort()
      {
          testSort(0);
      }
      void testHeapSort()
      {
          testSort(2);
      }
      void testMerge()
      {
          DBGLOG("testMerge");
          init();
          Owned<IPropertyTree> splitterNode = createPTree();
          Owned <IRoxieServerActivityFactory> factory = createRoxieServerMergeActivityFactory(1, 1, *queryFactory, mergeActivityTestFactory, TAKmerge, *splitterNode);
          factory->setInput(0,0,0);
          factory->setInput(1,0,0);
          Owned <IRoxieServerActivity> activity = factory->createActivity(ctx, NULL);
          const char * test[] = { NULL, NULL };
          const char * test12345[] = { "1", "2", "3", "4", "5", NULL, NULL };
          const char * test1122334455[] = { "1", "1", "2", "2", "3", "3", "4", "4", "5", "5", NULL, NULL };
          const char * test11111[] = { "1", "1", "1", "1", "1", NULL, NULL };
          const char * test1111111111[] = { "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", NULL, NULL };
          const char * test11111_12345[] = { "1", "1", "1", "1", "1", NULL, "1", "2", "3", "4", "5", NULL, NULL };
          const char * test1111112345[] = { "1", "1", "1", "1", "1", "1", "2", "3", "4", "5", NULL, NULL };
          const char * test11111111111122334455[] = { "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "1", "2", "2", "3", "3", "4", "4", "5", "5", NULL, NULL };
          
          testActivity(activity, test, test, test);
          testActivity(activity, test12345, test, test12345);
          testActivity(activity, test, test12345, test12345);
          testActivity(activity, test12345, test12345, test1122334455);
          testActivity(activity, test11111, test, test11111);
          testActivity(activity, test, test11111, test11111);
          testActivity(activity, test11111, test11111, test1111111111);
          testActivity(activity, test11111_12345, test, test1111112345);
          testActivity(activity, test11111_12345, test11111_12345, test11111111111122334455);
  
          // Should really test WHICH side gets kept...
          // Should test with more than 2 inputs...
          DBGLOG("testMerge done");
      }
  
      void testMergeDedup()
      {
          DBGLOG("testMergeDedup");
          init();
          MergeActivityTest::isDedup = true;
          Owned<IPropertyTree> splitterNode = createPTree();
          Owned <IRoxieServerActivityFactory> factory = createRoxieServerMergeActivityFactory(1, 1, *queryFactory, mergeActivityTestFactory, TAKmerge, *splitterNode);
          factory->setInput(0,0,0);
          factory->setInput(1,0,0);
          Owned <IRoxieServerActivity> activity = factory->createActivity(ctx, NULL);
          const char * test[] = { NULL, NULL };
          const char * test12345[] = { "1", "2", "3", "4", "5", NULL, NULL };
          const char * test11111[] = { "1", "1", "1", "1", "1", NULL, NULL };
          const char * test11111_12345[] = { "1", "1", "1", "1", "1", NULL, "1", "2", "3", "4", "5", NULL, NULL };
          const char * test1111112345[] = { "1", "1", "1", "1", "1", "1", "2", "3", "4", "5", NULL, NULL };
          
          testActivity(activity, test11111, test, test11111);   // No dedup within a stream
          testActivity(activity, test11111, test11111, test11111);   // No dedup within a stream
          testActivity(activity, test, test11111, test11111);
          testActivity(activity, test, test, test);
          testActivity(activity, test12345, test, test12345);
          testActivity(activity, test, test12345, test12345);
          testActivity(activity, test12345, test12345, test12345);
          testActivity(activity, test11111_12345, test, test1111112345);
          testActivity(activity, test11111_12345, test11111_12345, test1111112345);
  
          // Should really test WHICH side gets kept...
          // Should test with more than 2 inputs...
          DBGLOG("testMergeDedup done");
      }
  
      void doTestSplitter(unsigned numOutputs)
      {
          DBGLOG("testSplit %d", numOutputs);
          init();
          Owned<IPropertyTree> splitterNode = createPTree();
          Owned <IRoxieServerActivityFactory> factory = createRoxieServerThroughSpillActivityFactory(*queryFactory, splitActivityTestFactory, numOutputs, *splitterNode);
          factory->setInput(0,0,0);
          Owned <IRoxieServerActivity> activity = factory->createActivity(ctx, NULL);
  
          const char * test[] = { NULL, NULL };
          const char * test12345[] = { "1", "2", "3", "4", "5", NULL, NULL };
  
          unsigned start = msTick();
          testSplitActivity(factory, test, test, numOutputs, 0);
          testSplitActivity(factory, test12345, test12345, numOutputs, 0);
          testSplitActivity(factory, test12345, test12345, numOutputs, 100);
          unsigned elapsed = msTick() - start;
  
          DBGLOG("testSplit %d done in %dms", numOutputs, elapsed);
      }
  
      void testSplitter()
      {
          doTestSplitter(1);
          doTestSplitter(2);
          doTestSplitter(3);
          doTestSplitter(10);
      }
  
      void testMiscellaneous()
      {
          DBGLOG("sizeof(CriticalSection)=%u", (unsigned) sizeof(CriticalSection));
          DBGLOG("sizeof(SpinLock)=%u", (unsigned) sizeof(SpinLock));
          DBGLOG("sizeof(CJoinGroup)=%u", (unsigned) sizeof(CJoinGroup));
          ASSERT(sizeof(CJoinGroup) <= 120);
      }
  };
  
  CPPUNIT_TEST_SUITE_REGISTRATION( CcdServerTest );
  CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CcdServerTest, "CcdServerTest" );
  
  #endif
  