<commit_before>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "log.h"
#include <string.h>
#include <iostream>
#include <iomanip>
#include <string>
#include <unordered_map>
#include <vector>

#ifdef MULTITHREAD
#include <mutex>
#endif  // MULTITHREAD

namespace Log {
namespace Detail {

int verbosity = 0;
int maximumLogLevel = 0;

// The time at which logging was initialized; used so that log messages can have
// relative rather than absolute timestamps.
static uint64_t initTime = 0;

// The first level cache for fileLogLevel() - the most recent result returned.
static const char* mostRecentFile = nullptr;
static int mostRecentLevel = -1;

// The second level cache for fileLogLevel(), mapping filenames to log levels.
static std::unordered_map<const void*, int> logLevelCache;

// All log levels manually specified by the user.
static std::vector<std::string> debugSpecs;

std::ostream& operator<<(std::ostream& out, const Log::Detail::OutputLogPrefix& pfx) {
#ifdef CLOCK_MONOTONIC
    if (LOGGING(2)) {
        struct timespec ts;
        clock_gettime(CLOCK_MONOTONIC, &ts);
        uint64_t t = ts.tv_sec*1000000000UL + ts.tv_nsec - Log::Detail::initTime;
        t /= 1000000UL;  // millisec
        out << t/1000 << '.' << std::setw(3) << std::setfill('0') << t%1000 << ':'
            << std::setfill(' '); }
#endif
    if (LOGGING(1)) {
        const char *s = strrchr(pfx.fn, '/');
        const char *e = strrchr(pfx.fn, '.');
        s = s ? s + 1 : pfx.fn;
        if (e && e > s)
            out.write(s, e-s);
        else
            out << s;
        out << ':' << pfx.level << ':'; }
    return out;
}

static bool match(const char *pattern, const char *name) {
    const char *pend = pattern + strcspn(pattern, ",:");
    const char *pbackup = 0;
    while (1) {
        while (pattern < pend && *pattern == *name) {
            pattern++;
            name++; }
        if (pattern == pend) {
            if (!strcmp(name, ".cpp")) return true;
            return *name == 0; }
        if (*pattern++ != '*') return false;
        if (pattern == pend) return true;
        while (*name && *name != *pattern) {
            if (pbackup && *name == *pbackup) {
                pattern = pbackup;
                break; }
            name++; }
        pbackup = pattern;
    }
}

int uncachedFileLogLevel(const char* file) {
    if (auto* startOfFilename = strrchr(file, '/'))
        file = startOfFilename + 1;

    for (auto& spec : debugSpecs)
        for (auto* pattern = spec.c_str(); pattern; pattern = strchr(pattern, ',')) {
            while (*pattern == ',') pattern++;
            if (match(pattern, file))
                if (auto* level = strchr(pattern, ':'))
                    return atoi(level + 1); }

    // If there's no matching spec, compute a default from the global verbosity level.
    return verbosity > 0 ? verbosity - 1 : 0;
}

int fileLogLevel(const char* file) {
#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif

    // There are two layers of caching here. First, we cache the most recent
    // result we returned, to minimize expensive lookups in tight loops.
    if (mostRecentFile == file)
      return mostRecentLevel;

    mostRecentFile = file;

    // Second, we look up @file in a hash table mapping from pointers to log
    // levels. We expect to hit in this cache virtually all the time.
    auto logLevelCacheIter = logLevelCache.find(file);
    if (logLevelCacheIter != logLevelCache.end())
      return mostRecentLevel = logLevelCacheIter->second;

    // This is the slow path. We have to walk @debugSpecs to see if there are any
    // specs that match @file.
    mostRecentLevel = uncachedFileLogLevel(file);
    logLevelCache[file] = mostRecentLevel;
    return mostRecentLevel;
}

void invalidateCaches(int possibleNewMaxLogLevel) {
    mostRecentFile = nullptr;
    mostRecentLevel = 0;
    logLevelCache.clear();
    maximumLogLevel = std::max(maximumLogLevel, possibleNewMaxLogLevel);
}

}  // namespace Detail

void addDebugSpec(const char* spec) {
#ifdef CLOCK_MONOTONIC
    if (!Detail::initTime) {
        struct timespec ts;
        clock_gettime(CLOCK_MONOTONIC, &ts);
        Detail::initTime = ts.tv_sec*1000000000UL + ts.tv_nsec; }
#endif

    // Validate @spec.
    bool ok = false;
    long maxLogLevelInSpec = 0;
    for (auto* pattern = strchr(spec, ':'); pattern; pattern = strchr(pattern, ':')) {
        ok = true;
        long level = strtol(pattern + 1, const_cast<char**>(&pattern), 10);
        if (*pattern && *pattern != ',') {
            ok = false;
            break; }
        maxLogLevelInSpec = std::max(maxLogLevelInSpec, level);
    }

    if (!ok) {
        std::cerr << "Invalid debug trace spec '" << spec << "'" << std::endl;
        return; }

#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif

    Detail::debugSpecs.push_back(spec);
    Detail::invalidateCaches(maxLogLevelInSpec);
}

void increaseVerbosity() {
#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif

    Detail::verbosity++;
    Detail::invalidateCaches(Detail::verbosity - 1);
}

}  // namespace Log
<commit_msg>Clean up and fix log metalogging<commit_after>/*
Copyright 2013-present Barefoot Networks, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

#include "log.h"
#include <string.h>
#include <iostream>
#include <iomanip>
#include <string>
#include <unordered_map>
#include <vector>

#ifdef MULTITHREAD
#include <mutex>
#endif  // MULTITHREAD

namespace Log {
namespace Detail {

int verbosity = 0;
int maximumLogLevel = 0;

// The time at which logging was initialized; used so that log messages can have
// relative rather than absolute timestamps.
static uint64_t initTime = 0;

// The first level cache for fileLogLevel() - the most recent result returned.
static const char* mostRecentFile = nullptr;
static int mostRecentLevel = -1;

// The second level cache for fileLogLevel(), mapping filenames to log levels.
static std::unordered_map<const void*, int> logLevelCache;

// All log levels manually specified by the user.
static std::vector<std::string> debugSpecs;

std::ostream& operator<<(std::ostream& out, const Log::Detail::OutputLogPrefix& pfx) {
#ifdef CLOCK_MONOTONIC
    if (LOGGING(2)) {
        struct timespec ts;
        clock_gettime(CLOCK_MONOTONIC, &ts);
        uint64_t t = ts.tv_sec*1000000000UL + ts.tv_nsec - Log::Detail::initTime;
        t /= 1000000UL;  // millisec
        out << t/1000 << '.' << std::setw(3) << std::setfill('0') << t%1000 << ':'
            << std::setfill(' '); }
#endif
    if (LOGGING(1)) {
        const char *s = strrchr(pfx.fn, '/');
        const char *e = strrchr(pfx.fn, '.');
        s = s ? s + 1 : pfx.fn;
        if (e && e > s)
            out.write(s, e-s);
        else
            out << s;
        out << ':' << pfx.level << ':'; }
    return out;
}

static bool match(const char *pattern, const char *name) {
    const char *pend = pattern + strcspn(pattern, ",:");
    const char *pbackup = 0;
    while (1) {
        while (pattern < pend && *pattern == *name) {
            pattern++;
            name++; }
        if (pattern == pend) {
            if (!strcmp(name, ".cpp")) return true;
            return *name == 0; }
        if (*pattern++ != '*') return false;
        if (pattern == pend) return true;
        while (*name && *name != *pattern) {
            if (pbackup && *name == *pbackup) {
                pattern = pbackup;
                break; }
            name++; }
        pbackup = pattern;
    }
}

int uncachedFileLogLevel(const char* file) {
    if (auto* startOfFilename = strrchr(file, '/'))
        file = startOfFilename + 1;

    for (auto& spec : debugSpecs)
        for (auto* pattern = spec.c_str(); pattern; pattern = strchr(pattern, ',')) {
            while (*pattern == ',') pattern++;
            if (match(pattern, file))
                if (auto* level = strchr(pattern, ':'))
                    return atoi(level + 1); }

    // If there's no matching spec, compute a default from the global verbosity level,
    // except for THIS file
    if (!strcmp(file, "log.cpp")) return 0;
    return verbosity > 0 ? verbosity - 1 : 0;
}

int fileLogLevel(const char* file) {
#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif

    // There are two layers of caching here. First, we cache the most recent
    // result we returned, to minimize expensive lookups in tight loops.
    if (mostRecentFile == file)
      return mostRecentLevel;

    mostRecentFile = file;

    // Second, we look up @file in a hash table mapping from pointers to log
    // levels. We expect to hit in this cache virtually all the time.
    auto logLevelCacheIter = logLevelCache.find(file);
    if (logLevelCacheIter != logLevelCache.end())
      return mostRecentLevel = logLevelCacheIter->second;

    // This is the slow path. We have to walk @debugSpecs to see if there are any
    // specs that match @file.
    mostRecentLevel = uncachedFileLogLevel(file);
    logLevelCache[file] = mostRecentLevel;
    return mostRecentLevel;
}

void invalidateCaches(int possibleNewMaxLogLevel) {
    mostRecentFile = nullptr;
    mostRecentLevel = 0;
    logLevelCache.clear();
    maximumLogLevel = std::max(maximumLogLevel, possibleNewMaxLogLevel);
}

}  // namespace Detail

void addDebugSpec(const char* spec) {
#ifdef CLOCK_MONOTONIC
    if (!Detail::initTime) {
        struct timespec ts;
        clock_gettime(CLOCK_MONOTONIC, &ts);
        Detail::initTime = ts.tv_sec*1000000000UL + ts.tv_nsec; }
#endif

    // Validate @spec.
    bool ok = false;
    long maxLogLevelInSpec = 0;
    for (auto* pattern = strchr(spec, ':'); pattern; pattern = strchr(pattern, ':')) {
        ok = true;
        long level = strtol(pattern + 1, const_cast<char**>(&pattern), 10);
        if (*pattern && *pattern != ',') {
            ok = false;
            break; }
        maxLogLevelInSpec = std::max(maxLogLevelInSpec, level);
    }

    if (!ok) {
        std::cerr << "Invalid debug trace spec '" << spec << "'" << std::endl;
        return; }

#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif

    Detail::debugSpecs.push_back(spec);
    Detail::invalidateCaches(maxLogLevelInSpec);
}

void increaseVerbosity() {
#ifdef MULTITHREAD
    static std::mutex lock;
    std::lock_guard<std::mutex> acquire(lock);
#endif
#ifdef CLOCK_MONOTONIC
    if (!Detail::initTime) {
        struct timespec ts;
        clock_gettime(CLOCK_MONOTONIC, &ts);
        Detail::initTime = ts.tv_sec*1000000000UL + ts.tv_nsec; }
#endif

    Detail::verbosity++;
    Detail::invalidateCaches(Detail::verbosity - 1);
}

}  // namespace Log
<|endoftext|>


<commit_before>/*
 * CliqueConn.cpp
 *
 *  Created on: Sep 8, 2011
 *      Author: gkenyon
 */

#include "CliqueConn.hpp"
int pvpatch_update_clique(int nk, float* RESTRICT v, float a, float* RESTRICT w);
int pvpatch_update_clique2(int nk, float* RESTRICT v, float a, float* RESTRICT w, float* RESTRICT m);

namespace PV {

CliqueConn::CliqueConn(const char * name, HyPerCol * hc, HyPerLayer * pre,
      HyPerLayer * post, ChannelType channel, const char * filename,
      InitWeights *weightInit)
{
   CliqueConn::initialize_base();
   CliqueConn::initialize(name, hc, pre, post, channel, filename, weightInit);
};

int CliqueConn::initialize_base(){
   cliqueSize = 1;
   KernelConn::initialize_base();
   return PV_SUCCESS;
}

int CliqueConn::initialize(const char * name, HyPerCol * hc, HyPerLayer * pre,
      HyPerLayer * post, ChannelType channel, const char * filename,
      InitWeights *weightInit){
   KernelConn::initialize(name, hc, pre, post, channel, filename, weightInit);
   PVParams * params = parent->parameters();
   cliqueSize = params->value(name, "cliqueSize", 1, true);
   return PV_SUCCESS;
}

int CliqueConn::updateState(float time, float dt)
{
   int status = KernelConn::updateState(time, dt);
   assert(status == PV_SUCCESS);
   return PV_SUCCESS;
};

int CliqueConn::update_dW(int arborId)
{
   const PVLayerLoc * preLoc = this->getPre()->getLayerLoc();
   const int nfPre = preLoc->nf;
   //const int nxPre = preLoc->nx;
   //const int nyPre = preLoc->ny;
   const int nxPreExt = preLoc->nx + 2 * preLoc->nb;
   const int nyPreExt = preLoc->ny + 2 * preLoc->nb;

   int syPostExt = post->getLayerLoc()->nf
         * (post->getLayerLoc()->nx + 2 * post->getLayerLoc()->nb); // compute just once

   // if pre and post are the same layers, make a clone of size PVPatch to hold temporary activity values
   // in order to eliminate generalize self-interactions
   bool self_flag = this->getPre() == this->getPost();
   pvdata_t * a_post_mask = NULL;
   const int a_post_size = nfp * nxp * nyp;
   a_post_mask = (pvdata_t *) calloc(a_post_size, sizeof(pvdata_t));
   assert(a_post_mask != NULL);
   // get linear index of cell at center of patch for self_flag == true
   const int k_post_self = (int) (a_post_size / 2);  // if self_flag == true, a_post_size should be odd

   int delay = getDelay(arborId);

   // gather active indices in extended layer
   // hard to pre-compute at HyPerLayer level because of variable delays
   int numActiveExt = 0;
   unsigned int * activeExt = (unsigned int *) calloc(this->getPre()->getNumExtended(),
         sizeof(int));
   assert(activeExt != NULL);
   const pvdata_t * aPre = this->getPre()->getLayerData(delay);
   for (int kPreExt = 0; kPreExt < this->getPre()->getNumExtended(); kPreExt++) {
      if (aPre[kPreExt] == 0) continue;
      activeExt[numActiveExt++] = kPreExt;
   }

   // calc dimensions of wPostPatch
   // TODO: following is copied from calculation of wPostPatches and should be pre-calculated and stored there in HyPerConn::wPostPatches
   // TODO: following should be implemented as HyPerConn::calcPostPatchSize
   //const PVLayer * lPre = clayer;
   const float xScaleDiff = this->getPost()->getXScale() - this->getPre()->getXScale();
   const float yScaleDiff = this->getPost()->getYScale() - this->getPre()->getYScale();
   const float powXScale = pow(2.0f, (float) xScaleDiff);
   const float powYScale = pow(2.0f, (float) yScaleDiff);
   //const int prePad = lPre->loc.nb;
   const int nxPostPatch = (int) (this->xPatchSize() * powXScale); // TODO: store in HyPerConn::wPostPatches
   const int nyPostPatch = (int) (this->yPatchSize() * powYScale); // TODO: store in HyPerConn::wPostPatches
   //const int nfPostPatch = nf;

   // clique dimensions
   // a new set of cliques is centered on each pre-synaptic cell with radius nzPostPatch/2
   // TODO: precompute clique dimensions during CliqueConn::initialize
   int nyCliqueRadius = (int) (nyPostPatch / 2);
   int nxCliqueRadius = (int) (nxPostPatch / 2);
   int cliquePatchSize = (2 * nxCliqueRadius + 1) * (2 * nyCliqueRadius + 1) * nfPre;
   //int numKernels = conn->numDataPatches();  // per arbor?
   int numCliques = pow(cliquePatchSize, cliqueSize - 1);
   assert(numCliques == this->numberOfAxonalArborLists());

   // loop over all products of cliqueSize active presynaptic cells
   // outer loop is over presynaptic cells, each of which defines the center of a cliquePatch
   // inner loop is over all combinations of clique cells within cliquePatch boundaries, which may be shrunken
   // TODO: pre-allocate cliqueActiveIndices as CliqueConn::cliquePatchSize member variable
   int * cliqueActiveIndices = (int *) calloc(cliquePatchSize, sizeof(int));
   assert(cliqueActiveIndices != NULL);
   for (int kPreActive = 0; kPreActive < numActiveExt; kPreActive++) {
      int kPreExt = activeExt[kPreActive];

      // get indices of active elements in clique radius
      // watch out for shrunken patches!
      int numActiveElements = 0;
      int kxPreExt = kxPos(kPreExt, nxPreExt, nyPreExt, nfPre);
      int kyPreExt = kyPos(kPreExt, nxPreExt, nyPreExt, nfPre);
      if (cliqueSize > 1) {
         for (int kyCliqueExt = ((kyPreExt - nyCliqueRadius) > 0 ? (kyPreExt - nyCliqueRadius) : 0);
               kyCliqueExt < ((kyPreExt + nyCliqueRadius) <= nyPreExt ? (kyPreExt + nyCliqueRadius) : nyPreExt); kyCliqueExt++) {
            //if (kyCliqueExt < 0 || kyCliqueExt > nyPreExt) continue;
            for (int kxCliqueExt = ((kxPreExt - nxCliqueRadius) > 0 ? (kxPreExt - nxCliqueRadius) : 0);
                  kxCliqueExt < ((kxPreExt + nxCliqueRadius) <= nxPreExt ? (kxPreExt + nxCliqueRadius) : nxPreExt); kxCliqueExt++) {
               //if (kyCliqueExt < 0 || kyCliqueExt > nxPreExt) continue;
               for (int kfCliqueExt = 0; kfCliqueExt < nfPre; kfCliqueExt++) {
                  int kCliqueExt = kIndex(kxCliqueExt, kyCliqueExt, kfCliqueExt, nxPreExt,
                        nyPreExt, nfPre);
                  if ((aPre[kCliqueExt] == 0) || (kCliqueExt == kPreExt)) continue;
                  cliqueActiveIndices[numActiveElements++] = kCliqueExt;
               }
            }
         }
      } // cliqueSize > 1
      else {
         cliqueActiveIndices[numActiveElements++] = kPreExt; // each cell is its own clique if cliqueSize == 1
      }
      if (numActiveElements < (cliqueSize-1)) continue;

      // loop over all active combinations of size=cliqueSize-1 in clique radius
      int numActiveCliques = pow(numActiveElements, cliqueSize - 1);
      for (int kClique = 0; kClique < numActiveCliques; kClique++) {

         //initialize a_post_tmp
         if (self_flag) {  // otherwise, a_post_mask is not modified and thus doesn't have to be updated
            for (int k_post = 0; k_post < a_post_size; k_post++) {
               a_post_mask[k_post] = 1;
            }
            a_post_mask[k_post_self] = 0;
         }

         // decompose kClique to compute product of active clique elements
         int arborNdx = 0;
         pvdata_t cliqueProd = aPre[kPreExt];
         int kResidue = kClique;
         int maxIndex = -1;
         for (int iProd = 0; iProd < cliqueSize - 1; iProd++) {
            int kPatchActive = (unsigned int) (kResidue
                  / pow(numActiveElements, cliqueSize - 1 - iProd - 1));

            // only apply each permutation of clique elements once, no element can contribute more than once
            if (kPatchActive <= maxIndex) {
               break;
            }
            else {
               maxIndex = kPatchActive;
            }
            int kCliqueExt = cliqueActiveIndices[kPatchActive];
            cliqueProd *= aPre[kCliqueExt];
            kResidue = kResidue
                  - kPatchActive * pow(numActiveElements, cliqueSize - 1 - iProd - 1);

            // compute arborIndex for this clique element
            int kxCliqueExt = kxPos(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kyCliqueExt = kyPos(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kfClique = featureIndex(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kxPatch = kxCliqueExt - kxPreExt + nxCliqueRadius;
            int kyPatch = kyCliqueExt - kyPreExt + nyCliqueRadius;
            unsigned int kArbor = kIndex(kxPatch, kyPatch, kfClique,
                  (2 * nxCliqueRadius + 1), (2*nyCliqueRadius + 1), nfPre);
            arborNdx += kArbor * pow(cliquePatchSize, cliqueSize - 1 - iProd - 1);
            if ((arborNdx < 0) || (arborNdx >= numCliques)){
                  assert((arborNdx >= 0) && (arborNdx < numCliques));
            }

            // remove self-interactions if pre == post
            if (self_flag){
               a_post_mask[kArbor] = 0;
            }

         } // iProd

         // receive weights input from clique (mostly copied from superclass method)
         // PVAxonalArbor * arbor = this->axonalArbor(kPreExt, arborNdx);
         PVPatch * dWPatch = dwPatches[arborNdx][kPreExt]; // arbor->plasticIncr;
         size_t postOffset = getAPostOffset(kPreExt, arborNdx);
         const float * aPost = &post->getLayerData()[postOffset];

         const pvdata_t * dWStart = this->getPIncrDataStart(arborNdx);
         int kernelIndex = this->patchIndexToKernelIndex(kPreExt);
         const pvdata_t * dW_head = &(dWStart[a_post_size*kernelIndex]);
         size_t dW_offset = dWPatch->data - dW_head;

         // WARNING - assumes weight and GSyn patches from task same size
         //         - assumes patch stride sf is 1

         int nkPatch = nfp * dWPatch->nx;
         int nyPatch = dWPatch->ny;
         int syPatch = syp;

         // TODO - unroll
         for (int y = 0; y < nyPatch; y++) {
            pvpatch_update_clique2(
                  nkPatch,
                  (float *) (dWPatch->data + y * syPatch),
                  cliqueProd,
                  (float *) (aPost + y*syPostExt),
                  (float *) (a_post_mask + dW_offset + y * syPatch));
               }

            } // kClique
   } // kPreActive
   free(cliqueActiveIndices);
   free(activeExt);
   free(a_post_mask);
   return PV_BREAK;

}
;
// calc_dW

int CliqueConn::updateWeights(int arborId)
{
   int status = KernelConn::updateWeights(arborId);
   assert((status == PV_SUCCESS) || (status == PV_BREAK));
   return PV_BREAK;

}
;
// updateWeights

/*
 int CliqueConn::normalizeWeights(PVPatch ** patches, int numPatches, int arborId){
 return PV_CONTINUE;};
 */

}//  namespace PV

int pvpatch_update_clique(int nk, float* RESTRICT dW, float aPre, float* RESTRICT aPost)
{
   int k;
   int err = 0;
   for (k = 0; k < nk; k++) {
      dW[k] += aPre * aPost[k];
   }
   return err;
}

int pvpatch_update_clique2(int nk, float* RESTRICT dW, float aPre, float* RESTRICT aPost, float* RESTRICT a_mask)
{
   int k;
   int err = 0;
   for (k = 0; k < nk; k++) {
      dW[k] += aPre * aPost[k] * a_mask[k];
   }
   return err;
}

<commit_msg>added code to eliminate generalized self interactions in the rcvSynapticInput method<commit_after>/*
 * CliqueConn.cpp
 *
 *  Created on: Sep 8, 2011
 *      Author: gkenyon
 */

#include "CliqueConn.hpp"
int pvpatch_update_clique(int nk, float* RESTRICT v, float a, float* RESTRICT w);
int pvpatch_update_clique2(int nk, float* RESTRICT v, float a, float* RESTRICT w, float* RESTRICT m);

namespace PV {

CliqueConn::CliqueConn(const char * name, HyPerCol * hc, HyPerLayer * pre,
      HyPerLayer * post, ChannelType channel, const char * filename,
      InitWeights *weightInit)
{
   CliqueConn::initialize_base();
   CliqueConn::initialize(name, hc, pre, post, channel, filename, weightInit);
};

int CliqueConn::initialize_base(){
   cliqueSize = 1;
   KernelConn::initialize_base();
   return PV_SUCCESS;
}

int CliqueConn::initialize(const char * name, HyPerCol * hc, HyPerLayer * pre,
      HyPerLayer * post, ChannelType channel, const char * filename,
      InitWeights *weightInit){
   KernelConn::initialize(name, hc, pre, post, channel, filename, weightInit);
   PVParams * params = parent->parameters();
   cliqueSize = params->value(name, "cliqueSize", 1, true);
   return PV_SUCCESS;
}

int CliqueConn::updateState(float time, float dt)
{
   int status = KernelConn::updateState(time, dt);
   assert(status == PV_SUCCESS);
   return PV_SUCCESS;
};

int CliqueConn::update_dW(int arborId)
{
   const PVLayerLoc * preLoc = this->getPre()->getLayerLoc();
   const int nfPre = preLoc->nf;
   //const int nxPre = preLoc->nx;
   //const int nyPre = preLoc->ny;
   const int nxPreExt = preLoc->nx + 2 * preLoc->nb;
   const int nyPreExt = preLoc->ny + 2 * preLoc->nb;

   int syPostExt = post->getLayerLoc()->nf
         * (post->getLayerLoc()->nx + 2 * post->getLayerLoc()->nb); // compute just once

   // if pre and post are the same layers, make a clone of size PVPatch to hold temporary activity values
   // in order to eliminate generalize self-interactions
   bool self_flag = this->getPre() == this->getPost();
   pvdata_t * a_post_mask = NULL;
   const int a_post_size = nfp * nxp * nyp;
   a_post_mask = (pvdata_t *) calloc(a_post_size, sizeof(pvdata_t));
   assert(a_post_mask != NULL);
   // get linear index of cell at center of patch for self_flag == true
   const int k_post_self = (int) (a_post_size / 2);  // if self_flag == true, a_post_size should be odd
   for (int k_post = 0; k_post < a_post_size; k_post++) {
      a_post_mask[k_post] = 1;
   }

   int delay = getDelay(arborId);

   // gather active indices in extended layer
   // hard to pre-compute at HyPerLayer level because of variable delays
   int numActiveExt = 0;
   unsigned int * activeExt = (unsigned int *) calloc(this->getPre()->getNumExtended(),
         sizeof(int));
   assert(activeExt != NULL);
   const pvdata_t * aPre = this->getPre()->getLayerData(delay);
   for (int kPreExt = 0; kPreExt < this->getPre()->getNumExtended(); kPreExt++) {
      if (aPre[kPreExt] == 0) continue;
      activeExt[numActiveExt++] = kPreExt;
   }

   // calc dimensions of wPostPatch
   // TODO: following is copied from calculation of wPostPatches and should be pre-calculated and stored there in HyPerConn::wPostPatches
   // TODO: following should be implemented as HyPerConn::calcPostPatchSize
   //const PVLayer * lPre = clayer;
   const float xScaleDiff = this->getPost()->getXScale() - this->getPre()->getXScale();
   const float yScaleDiff = this->getPost()->getYScale() - this->getPre()->getYScale();
   const float powXScale = pow(2.0f, (float) xScaleDiff);
   const float powYScale = pow(2.0f, (float) yScaleDiff);
   //const int prePad = lPre->loc.nb;
   const int nxPostPatch = (int) (this->xPatchSize() * powXScale); // TODO: store in HyPerConn::wPostPatches
   const int nyPostPatch = (int) (this->yPatchSize() * powYScale); // TODO: store in HyPerConn::wPostPatches
   //const int nfPostPatch = nf;

   // clique dimensions
   // a new set of cliques is centered on each pre-synaptic cell with radius nzPostPatch/2
   // TODO: precompute clique dimensions during CliqueConn::initialize
   int nyCliqueRadius = (int) (nyPostPatch / 2);
   int nxCliqueRadius = (int) (nxPostPatch / 2);
   int cliquePatchSize = (2 * nxCliqueRadius + 1) * (2 * nyCliqueRadius + 1) * nfPre;
   //int numKernels = conn->numDataPatches();  // per arbor?
   int numCliques = pow(cliquePatchSize, cliqueSize - 1);
   assert(numCliques == this->numberOfAxonalArborLists());

   // loop over all products of cliqueSize active presynaptic cells
   // outer loop is over presynaptic cells, each of which defines the center of a cliquePatch
   // inner loop is over all combinations of clique cells within cliquePatch boundaries, which may be shrunken
   // TODO: pre-allocate cliqueActiveIndices as CliqueConn::cliquePatchSize member variable
   int * cliqueActiveIndices = (int *) calloc(cliquePatchSize, sizeof(int));
   assert(cliqueActiveIndices != NULL);
   for (int kPreActive = 0; kPreActive < numActiveExt; kPreActive++) {
      int kPreExt = activeExt[kPreActive];

      // get indices of active elements in clique radius
      // watch out for shrunken patches!
      int numActiveElements = 0;
      int kxPreExt = kxPos(kPreExt, nxPreExt, nyPreExt, nfPre);
      int kyPreExt = kyPos(kPreExt, nxPreExt, nyPreExt, nfPre);
      if (cliqueSize > 1) {
         for (int kyCliqueExt = ((kyPreExt - nyCliqueRadius) > 0 ? (kyPreExt - nyCliqueRadius) : 0);
               kyCliqueExt < ((kyPreExt + nyCliqueRadius) <= nyPreExt ? (kyPreExt + nyCliqueRadius) : nyPreExt); kyCliqueExt++) {
            //if (kyCliqueExt < 0 || kyCliqueExt > nyPreExt) continue;
            for (int kxCliqueExt = ((kxPreExt - nxCliqueRadius) > 0 ? (kxPreExt - nxCliqueRadius) : 0);
                  kxCliqueExt < ((kxPreExt + nxCliqueRadius) <= nxPreExt ? (kxPreExt + nxCliqueRadius) : nxPreExt); kxCliqueExt++) {
               //if (kyCliqueExt < 0 || kyCliqueExt > nxPreExt) continue;
               for (int kfCliqueExt = 0; kfCliqueExt < nfPre; kfCliqueExt++) {
                  int kCliqueExt = kIndex(kxCliqueExt, kyCliqueExt, kfCliqueExt, nxPreExt,
                        nyPreExt, nfPre);
                  if ((aPre[kCliqueExt] == 0) || (kCliqueExt == kPreExt)) continue;
                  cliqueActiveIndices[numActiveElements++] = kCliqueExt;
               }
            }
         }
      } // cliqueSize > 1
      else {
         cliqueActiveIndices[numActiveElements++] = kPreExt; // each cell is its own clique if cliqueSize == 1
      }
      if (numActiveElements < (cliqueSize-1)) continue;

      // loop over all active combinations of size=cliqueSize-1 in clique radius
      int numActiveCliques = pow(numActiveElements, cliqueSize - 1);
      for (int kClique = 0; kClique < numActiveCliques; kClique++) {

         //initialize a_post_tmp
         if (self_flag) {  // otherwise, a_post_mask is not modified and thus doesn't have to be updated
            for (int k_post = 0; k_post < a_post_size; k_post++) {
               a_post_mask[k_post] = 1;
            }
            a_post_mask[k_post_self] = 0;
         }

         // decompose kClique to compute product of active clique elements
         int arborNdx = 0;
         pvdata_t cliqueProd = aPre[kPreExt];
         int kResidue = kClique;
         int maxIndex = -1;
         for (int iProd = 0; iProd < cliqueSize - 1; iProd++) {
            int kPatchActive = (unsigned int) (kResidue
                  / pow(numActiveElements, cliqueSize - 1 - iProd - 1));

            // only apply each permutation of clique elements once, no element can contribute more than once
            if (kPatchActive <= maxIndex) {
               break;
            }
            else {
               maxIndex = kPatchActive;
            }
            int kCliqueExt = cliqueActiveIndices[kPatchActive];
            cliqueProd *= aPre[kCliqueExt];
            kResidue = kResidue
                  - kPatchActive * pow(numActiveElements, cliqueSize - 1 - iProd - 1);

            // compute arborIndex for this clique element
            int kxCliqueExt = kxPos(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kyCliqueExt = kyPos(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kfClique = featureIndex(kCliqueExt, nxPreExt, nyPreExt, nfPre);
            int kxPatch = kxCliqueExt - kxPreExt + nxCliqueRadius;
            int kyPatch = kyCliqueExt - kyPreExt + nyCliqueRadius;
            unsigned int kArbor = kIndex(kxPatch, kyPatch, kfClique,
                  (2 * nxCliqueRadius + 1), (2*nyCliqueRadius + 1), nfPre);
            arborNdx += kArbor * pow(cliquePatchSize, cliqueSize - 1 - iProd - 1);
            if ((arborNdx < 0) || (arborNdx >= numCliques)){
                  assert((arborNdx >= 0) && (arborNdx < numCliques));
            }

            // remove self-interactions if pre == post
            if (self_flag){
               a_post_mask[kArbor] = 0;
            }

         } // iProd

         // receive weights input from clique (mostly copied from superclass method)
         // PVAxonalArbor * arbor = this->axonalArbor(kPreExt, arborNdx);
         PVPatch * dWPatch = dwPatches[arborNdx][kPreExt]; // arbor->plasticIncr;
         size_t postOffset = getAPostOffset(kPreExt, arborNdx);
         const float * aPost = &post->getLayerData()[postOffset];

         const pvdata_t * dWStart = this->getPIncrDataStart(arborNdx);
         int kernelIndex = this->patchIndexToKernelIndex(kPreExt);
         const pvdata_t * dW_head = &(dWStart[a_post_size*kernelIndex]);
         size_t dW_offset = dWPatch->data - dW_head;

         // WARNING - assumes weight and GSyn patches from task same size
         //         - assumes patch stride sf is 1

         int nkPatch = nfp * dWPatch->nx;
         int nyPatch = dWPatch->ny;
         int syPatch = syp;

         // TODO - unroll
         for (int y = 0; y < nyPatch; y++) {
            pvpatch_update_clique2(
                  nkPatch,
                  (float *) (dWPatch->data + y * syPatch),
                  cliqueProd,
                  (float *) (aPost + y*syPostExt),
                  (float *) (a_post_mask + dW_offset + y * syPatch));
               }

            } // kClique
   } // kPreActive
   free(cliqueActiveIndices);
   free(activeExt);
   free(a_post_mask);
   return PV_BREAK;

}
;
// calc_dW

int CliqueConn::updateWeights(int arborId)
{
   int status = KernelConn::updateWeights(arborId);
   assert((status == PV_SUCCESS) || (status == PV_BREAK));
   return PV_BREAK;

}
;
// updateWeights

/*
 int CliqueConn::normalizeWeights(PVPatch ** patches, int numPatches, int arborId){
 return PV_CONTINUE;};
 */

}//  namespace PV

int pvpatch_update_clique(int nk, float* RESTRICT dW, float aPre, float* RESTRICT aPost)
{
   int k;
   int err = 0;
   for (k = 0; k < nk; k++) {
      dW[k] += aPre * aPost[k];
   }
   return err;
}

int pvpatch_update_clique2(int nk, float* RESTRICT dW, float aPre, float* RESTRICT aPost, float* RESTRICT a_mask)
{
   int k;
   int err = 0;
   for (k = 0; k < nk; k++) {
      dW[k] += aPre * aPost[k] * a_mask[k];
   }
   return err;
}

<|endoftext|>


<commit_before>#include <arpa/inet.h>    // inet_ntoa
#include <fstream>
#include <iostream>
#include <netdb.h>        // gethostbyname
#include <netinet/in.h>   // htonl, htons, inet_ntoa
#include <netinet/tcp.h>  // SO_REUSEADDR
#include <pthread.h>
#include <sstream>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <strings.h>      // bzero
#include <sys/types.h>    // socket, bind
#include <sys/socket.h>   // socket, bind, listen, inet_ntoa
#include <sys/uio.h>      // writev
#include <sys/time.h>
#include <unistd.h>       // read, write, close

using namespace std;

struct threadData
{
  int sd;
};

#define BUF_SIZE 16384
#define PORT 80
//
// IF_FALSE_RETURN
//
// Macro to convieniently add basic error handling
//
#define IF_FALSE_RETURN(test, msg) \
  if (!(test)) \
  { \
    printf("%s\n", msg); \
    return -1; \
  }

// ConvertParameterToInt
//
// several of our parameters are expected to be well formed numbers
// since no negative values are expected, return -1 to indicate an error.
//
int ConvertParameterToInt(char* value)
{
  char* endptr = NULL;
  long int result = strtol(value, &endptr, 0);
  if (*value == '\0' || *endptr != '\0')
    return -1;

  return (int)result;
}

void* openAndSendFile(void* whatever)
{

  //  If file requested does not exist, return 404 Not Found code with custom File Not Found page
  //
  //
  //
  //  If HTTP request is for SecretFile.html, return 401 Unauthorized
  //
  //
  //
  //  If request is for file that is above the directory structure where web server is running ( for example, "GET ../../../etc/passwd" ), return 403 Forbidden
  //
  //
  //
  //  if server cannot understand request return 400 Bad Request
  //
  //
  //

  threadData* data = (threadData*)whatever;
  int sd = data->sd;
  delete data;
  // After receiving GET request...
  char databuf[BUF_SIZE];
  stringstream ssrequest;

  // Repeat reading data from the client into databuf[BUF_SIZE].
  // Note that the read system call may return without reading
  // the entire data if the network is slow.
  int readed = read(sd, &databuf, BUF_SIZE);

  while(readed > 0)
  {
    ssrequest.write(databuf, readed);
    readed = read(sd, &databuf, BUF_SIZE);
  }

  printf("Printing ssrequest:\n");
  cout << ssrequest.str();

  // open file requested...

/*
  // and send with HTTP 200 OK code.
  int written = 0;
  while (written < sizeof(count))
  {
    int bytesWritten = write(sd, &count, sizeof(count));
    if (bytesWritten == -1)
    {
      printf("write failed\n");
      exit(1);
    }

    written += bytesWritten;
  }

  if (close(sd) != 0)
  {
      printf("close failed\n");
      exit(1);
  }*/
}

int main(int argc, char** argv)
{
  //wait for connection and HTTP GET request (you may do this single threaded or multi-threaded)

  // Declare and initialize sockaddr_in structure
  sockaddr_in acceptSockAddr;
  bzero((char*)&acceptSockAddr, sizeof(acceptSockAddr));
  acceptSockAddr.sin_family      = AF_INET;
  acceptSockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
  acceptSockAddr.sin_port        = htons(port);

  // Open a stream-oriented socket with the Internet address family.
  int serverSd = socket(AF_INET, SOCK_STREAM, 0);
  IF_FALSE_RETURN(serverSd != -1, "socket failed to create file descriptor");

  // Set the SO_REUSEADDR option.
  //
  // Note: this option is useful to prompt OS to release the server port
  // as soon as your server process is terminated.
  const int on = 1;
  IF_FALSE_RETURN(setsockopt(serverSd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(int)) != -1, "setsockopt failed");

  // Bind this socket to its local address.
  IF_FALSE_RETURN(bind(serverSd, (sockaddr*)&acceptSockAddr, sizeof(acceptSockAddr)) != -1, "bind failed");

  // Instruct the operating system to listen to up to 8 connection requests from clients at a time
  IF_FALSE_RETURN(listen(serverSd, 8) != -1, "listen failed");

  printf("Server: listening on port %d\n", port);

  // Receive a request from a client by calling accept that will return a new socket specific to this connection request.
  sockaddr_in newSockAddr;
  socklen_t newSockAddrSize = sizeof(newSockAddr);
  int newSd = accept(serverSd, (sockaddr*)&newSockAddr, &newSockAddrSize);
  printf("Server: accepted connection\n");
  while (newSd != -1)
  {
    //Create a new thread
    threadData* data = new threadData();
    data->sd = newSd;

    pthread_t thread;
    if (pthread_create(&thread, NULL, openAndSendFile, data) != 0)
    {
      delete data;
      printf("pthread_create failed\n");
      return -1;
    }
    printf("Server: created thread\n");

    //After handling request, return to waiting for next request.

    newSd = accept(serverSd, (sockaddr*)&newSockAddr, &newSockAddrSize);
    printf("Server: accepted another connection\n");
  }

  IF_FALSE_RETURN(newSd != -1, "accept failed");
  return 0;
}<commit_msg>Another bugfix! If you haven't guessed yet, I'm compiling and coding on two different machines.<commit_after>#include <arpa/inet.h>    // inet_ntoa
#include <fstream>
#include <iostream>
#include <netdb.h>        // gethostbyname
#include <netinet/in.h>   // htonl, htons, inet_ntoa
#include <netinet/tcp.h>  // SO_REUSEADDR
#include <pthread.h>
#include <sstream>
#include <stdio.h>
#include <stdlib.h>
#include <string>
#include <strings.h>      // bzero
#include <sys/types.h>    // socket, bind
#include <sys/socket.h>   // socket, bind, listen, inet_ntoa
#include <sys/uio.h>      // writev
#include <sys/time.h>
#include <unistd.h>       // read, write, close

using namespace std;

struct threadData
{
  int sd;
};

#define BUF_SIZE 16384
#define PORT 80
//
// IF_FALSE_RETURN
//
// Macro to convieniently add basic error handling
//
#define IF_FALSE_RETURN(test, msg) \
  if (!(test)) \
  { \
    printf("%s\n", msg); \
    return -1; \
  }

// ConvertParameterToInt
//
// several of our parameters are expected to be well formed numbers
// since no negative values are expected, return -1 to indicate an error.
//
int ConvertParameterToInt(char* value)
{
  char* endptr = NULL;
  long int result = strtol(value, &endptr, 0);
  if (*value == '\0' || *endptr != '\0')
    return -1;

  return (int)result;
}

void* openAndSendFile(void* whatever)
{

  //  If file requested does not exist, return 404 Not Found code with custom File Not Found page
  //
  //
  //
  //  If HTTP request is for SecretFile.html, return 401 Unauthorized
  //
  //
  //
  //  If request is for file that is above the directory structure where web server is running ( for example, "GET ../../../etc/passwd" ), return 403 Forbidden
  //
  //
  //
  //  if server cannot understand request return 400 Bad Request
  //
  //
  //

  threadData* data = (threadData*)whatever;
  int sd = data->sd;
  delete data;
  // After receiving GET request...
  char databuf[BUF_SIZE];
  stringstream ssrequest;

  // Repeat reading data from the client into databuf[BUF_SIZE].
  // Note that the read system call may return without reading
  // the entire data if the network is slow.
  int readed = read(sd, &databuf, BUF_SIZE);

  while(readed > 0)
  {
    ssrequest.write(databuf, readed);
    readed = read(sd, &databuf, BUF_SIZE);
  }

  printf("Printing ssrequest:\n");
  cout << ssrequest.str();

  // open file requested...

/*
  // and send with HTTP 200 OK code.
  int written = 0;
  while (written < sizeof(count))
  {
    int bytesWritten = write(sd, &count, sizeof(count));
    if (bytesWritten == -1)
    {
      printf("write failed\n");
      exit(1);
    }

    written += bytesWritten;
  }

  if (close(sd) != 0)
  {
      printf("close failed\n");
      exit(1);
  }*/
}

int main(int argc, char** argv)
{
  //wait for connection and HTTP GET request (you may do this single threaded or multi-threaded)

  // Declare and initialize sockaddr_in structure
  sockaddr_in acceptSockAddr;
  bzero((char*)&acceptSockAddr, sizeof(acceptSockAddr));
  acceptSockAddr.sin_family      = AF_INET;
  acceptSockAddr.sin_addr.s_addr = htonl(INADDR_ANY);
  acceptSockAddr.sin_port        = htons(PORT);

  // Open a stream-oriented socket with the Internet address family.
  int serverSd = socket(AF_INET, SOCK_STREAM, 0);
  IF_FALSE_RETURN(serverSd != -1, "socket failed to create file descriptor");

  // Set the SO_REUSEADDR option.
  //
  // Note: this option is useful to prompt OS to release the server port
  // as soon as your server process is terminated.
  const int on = 1;
  IF_FALSE_RETURN(setsockopt(serverSd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(int)) != -1, "setsockopt failed");

  // Bind this socket to its local address.
  IF_FALSE_RETURN(bind(serverSd, (sockaddr*)&acceptSockAddr, sizeof(acceptSockAddr)) != -1, "bind failed");

  // Instruct the operating system to listen to up to 8 connection requests from clients at a time
  IF_FALSE_RETURN(listen(serverSd, 8) != -1, "listen failed");

  printf("Server: listening on port %d\n", port);

  // Receive a request from a client by calling accept that will return a new socket specific to this connection request.
  sockaddr_in newSockAddr;
  socklen_t newSockAddrSize = sizeof(newSockAddr);
  int newSd = accept(serverSd, (sockaddr*)&newSockAddr, &newSockAddrSize);
  printf("Server: accepted connection\n");
  while (newSd != -1)
  {
    //Create a new thread
    threadData* data = new threadData();
    data->sd = newSd;

    pthread_t thread;
    if (pthread_create(&thread, NULL, openAndSendFile, data) != 0)
    {
      delete data;
      printf("pthread_create failed\n");
      return -1;
    }
    printf("Server: created thread\n");

    //After handling request, return to waiting for next request.

    newSd = accept(serverSd, (sockaddr*)&newSockAddr, &newSockAddrSize);
    printf("Server: accepted another connection\n");
  }

  IF_FALSE_RETURN(newSd != -1, "accept failed");
  return 0;
}<|endoftext|>


<commit_before>/* -*- mode: C; c-basic-offset: 4; indent-tabs-mode: nil; -*- */
/*
 * Copyright (c) 2008  litl, LLC
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <config.h>
#include <glib.h>
#include <glib-object.h>
#include <gjs/gjs-module.h>
#include <util/glib.h>
#include <util/crash.h>

typedef struct _GjsUnitTestFixture GjsUnitTestFixture;

struct _GjsUnitTestFixture {
    JSContext *context;
    GjsContext *gjs_context;
};

static void
test_error_reporter(JSContext     *context,
                    const char    *message,
                    JSErrorReport *report)
{
    g_printerr("error reported by test: %s\n", message);
}

static void
_gjs_unit_test_fixture_begin (GjsUnitTestFixture *fixture)
{
    fixture->gjs_context = gjs_context_new ();
    fixture->context = (JSContext *) gjs_context_get_native_context (fixture->gjs_context);
    JS_BeginRequest(fixture->context);
    JS_SetErrorReporter(fixture->context, test_error_reporter);
}

static void
_gjs_unit_test_fixture_finish (GjsUnitTestFixture *fixture)
{
    JS_EndRequest(fixture->context);
    g_object_unref(fixture->gjs_context);
}

static void
gjstest_test_func_gjs_context_construct_destroy(void)
{
    GjsContext *context;

    /* Construct twice just to possibly a case where global state from
     * the first leaks.
     */
    context = gjs_context_new ();
    g_object_unref (context);

    context = gjs_context_new ();
    g_object_unref (context);
}

static void
gjstest_test_func_gjs_context_construct_eval(void)
{
    GjsContext *context;
    int estatus;
    GError *error = NULL;

    context = gjs_context_new ();
    if (!gjs_context_eval (context, "1+1", -1, "<input>", &estatus, &error))
        g_error ("%s", error->message);
    g_object_unref (context);
}

static void
gjstest_test_func_gjs_context_fixture(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;

    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);
}

#define N_ELEMS 15

static void
gjstest_test_func_gjs_jsapi_util_array(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;
    GjsRootedArray *array;
    int i;
    jsval value;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;

    array = gjs_rooted_array_new();

    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);

    for (i = 0; i < N_ELEMS; i++) {
        value = STRING_TO_JSVAL(JS_NewStringCopyZ(context, "abcdefghijk"));
        gjs_rooted_array_append(context, array, value);
    }

    JS_GC(JS_GetRuntime(context));

    for (i = 0; i < N_ELEMS; i++) {
        char *ascii;

        value = gjs_rooted_array_get(context, array, i);
        g_assert(JSVAL_IS_STRING(value));
        gjs_string_to_utf8(context, value, &ascii);
        g_assert(strcmp(ascii, "abcdefghijk") == 0);
        g_free(ascii);
    }

    gjs_rooted_array_free(context, array, TRUE);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);
}

#undef N_ELEMS

static void
gjstest_test_func_gjs_jsapi_util_string_js_string_utf8(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;

    const char *utf8_string = "\303\211\303\226 foobar \343\203\237";
    char *utf8_result;
    jsval js_string;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;
    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);

    g_assert(gjs_string_from_utf8(context, utf8_string, -1, &js_string) == JS_TRUE);
    g_assert(JSVAL_IS_STRING(js_string));
    g_assert(gjs_string_to_utf8(context, js_string, &utf8_result) == JS_TRUE);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);

    g_assert(g_str_equal(utf8_string, utf8_result));

    g_free(utf8_result);
}

static void
gjstest_test_func_gjs_stack_dump(void)
{
  GjsContext *context;

  /* TODO this test could be better - maybe expose dumpstack as a JS API
   * so that we have a JS stack to dump?  At least here we're getting some
   * coverage.
   */
  context = gjs_context_new();

  gjs_dumpstack();
  g_object_unref(context);
  gjs_dumpstack();
}

static void
gjstest_test_func_gjs_jsapi_util_error_throw(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;
    jsval exc, value, previous;
    char *s = NULL;
    int strcmp_result;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;
    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);
    /* Test that we can throw */

    gjs_throw(context, "This is an exception %d", 42);

    g_assert(JS_IsExceptionPending(context));

    exc = JSVAL_VOID;
    JS_GetPendingException(context, &exc);
    g_assert(!JSVAL_IS_VOID(exc));

    value = JSVAL_VOID;
    JS_GetProperty(context, JSVAL_TO_OBJECT(exc), "message",
                   &value);

    g_assert(JSVAL_IS_STRING(value));

    gjs_string_to_utf8(context, value, &s);
    g_assert(s != NULL);
    strcmp_result = strcmp(s, "This is an exception 42");
    free(s);
    if (strcmp_result != 0)
        g_error("Exception has wrong message '%s'", s);

    /* keep this around before we clear it */
    previous = exc;
    JS_AddValueRoot(context, &previous);

    JS_ClearPendingException(context);

    g_assert(!JS_IsExceptionPending(context));

    /* Check that we don't overwrite a pending exception */
    JS_SetPendingException(context, previous);

    g_assert(JS_IsExceptionPending(context));

    gjs_throw(context, "Second different exception %s", "foo");

    g_assert(JS_IsExceptionPending(context));

    exc = JSVAL_VOID;
    JS_GetPendingException(context, &exc);
    g_assert(!JSVAL_IS_VOID(exc));
    g_assert(JSVAL_TO_OBJECT(exc) == JSVAL_TO_OBJECT(previous));

    JS_RemoveValueRoot(context, &previous);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);
}

static void
gjstest_test_func_util_glib_strv_concat_null(void)
{
    char **ret;

    ret = gjs_g_strv_concat(NULL, 0);
    g_assert(ret != NULL);
    g_assert(ret[0] == NULL);

    g_strfreev(ret);
}

static void
gjstest_test_func_util_glib_strv_concat_pointers(void)
{
    char  *strv0[2] = {(char*)"foo", NULL};
    char  *strv1[1] = {NULL};
    char **strv2    = NULL;
    char  *strv3[2] = {(char*)"bar", NULL};
    char **stuff[4];
    char **ret;

    stuff[0] = strv0;
    stuff[1] = strv1;
    stuff[2] = strv2;
    stuff[3] = strv3;

    ret = gjs_g_strv_concat(stuff, 4);
    g_assert(ret != NULL);
    g_assert_cmpstr(ret[0], ==, strv0[0]);  /* same string */
    g_assert(ret[0] != strv0[0]);           /* different pointer */
    g_assert_cmpstr(ret[1], ==, strv3[0]);
    g_assert(ret[1] != strv3[0]);
    g_assert(ret[2] == NULL);

    g_strfreev(ret);
}

static void
gjstest_test_strip_shebang_no_advance_for_no_shebang(void)
{
    const char *script = "foo\nbar";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert_cmpstr(script, ==, stripped);
    g_assert(script_len == script_len_original);
    g_assert(line_number == 1);
}

static void
gjstest_test_strip_shebang_advance_for_shebang(void)
{
    const char *script = "#!foo\nbar";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert_cmpstr(stripped, ==, "bar");
    g_assert(script_len == 3);
    g_assert(line_number == 2);
}

static void
gjstest_test_strip_shebang_return_null_for_just_shebang(void)
{
    const char *script = "#!foo";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert(stripped == NULL);
    g_assert(script_len == 0);
    g_assert(line_number == -1);
}

int
main(int    argc,
     char **argv)
{
    gjs_crash_after_timeout(60*7); /* give the unit tests 7 minutes to complete */

    g_test_init(&argc, &argv, NULL);

    g_test_add_func("/gjs/context/construct/destroy", gjstest_test_func_gjs_context_construct_destroy);
    g_test_add_func("/gjs/context/construct/eval", gjstest_test_func_gjs_context_construct_eval);
    g_test_add_func("/gjs/context/fixture", gjstest_test_func_gjs_context_fixture);
    g_test_add_func("/gjs/jsapi/util/array", gjstest_test_func_gjs_jsapi_util_array);
    g_test_add_func("/gjs/jsapi/util/error/throw", gjstest_test_func_gjs_jsapi_util_error_throw);
    g_test_add_func("/gjs/jsapi/util/string/js/string/utf8", gjstest_test_func_gjs_jsapi_util_string_js_string_utf8);
    g_test_add_func("/gjs/jsutil/strip_shebang/no_shebang", gjstest_test_strip_shebang_no_advance_for_no_shebang);
    g_test_add_func("/gjs/jsutil/strip_shebang/have_shebang", gjstest_test_strip_shebang_advance_for_shebang);
    g_test_add_func("/gjs/jsutil/strip_shebang/only_shebang", gjstest_test_strip_shebang_return_null_for_just_shebang);
    g_test_add_func("/gjs/stack/dump", gjstest_test_func_gjs_stack_dump);
    g_test_add_func("/util/glib/strv/concat/null", gjstest_test_func_util_glib_strv_concat_null);
    g_test_add_func("/util/glib/strv/concat/pointers", gjstest_test_func_util_glib_strv_concat_pointers);

    g_test_run();

    return 0;
}
<commit_msg>gjs-tests: Remove fixture test<commit_after>/* -*- mode: C; c-basic-offset: 4; indent-tabs-mode: nil; -*- */
/*
 * Copyright (c) 2008  litl, LLC
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <config.h>
#include <glib.h>
#include <glib-object.h>
#include <gjs/gjs-module.h>
#include <util/glib.h>
#include <util/crash.h>

typedef struct _GjsUnitTestFixture GjsUnitTestFixture;

struct _GjsUnitTestFixture {
    JSContext *context;
    GjsContext *gjs_context;
};

static void
test_error_reporter(JSContext     *context,
                    const char    *message,
                    JSErrorReport *report)
{
    g_printerr("error reported by test: %s\n", message);
}

static void
_gjs_unit_test_fixture_begin (GjsUnitTestFixture *fixture)
{
    fixture->gjs_context = gjs_context_new ();
    fixture->context = (JSContext *) gjs_context_get_native_context (fixture->gjs_context);
    JS_BeginRequest(fixture->context);
    JS_SetErrorReporter(fixture->context, test_error_reporter);
}

static void
_gjs_unit_test_fixture_finish (GjsUnitTestFixture *fixture)
{
    JS_EndRequest(fixture->context);
    g_object_unref(fixture->gjs_context);
}

static void
gjstest_test_func_gjs_context_construct_destroy(void)
{
    GjsContext *context;

    /* Construct twice just to possibly a case where global state from
     * the first leaks.
     */
    context = gjs_context_new ();
    g_object_unref (context);

    context = gjs_context_new ();
    g_object_unref (context);
}

static void
gjstest_test_func_gjs_context_construct_eval(void)
{
    GjsContext *context;
    int estatus;
    GError *error = NULL;

    context = gjs_context_new ();
    if (!gjs_context_eval (context, "1+1", -1, "<input>", &estatus, &error))
        g_error ("%s", error->message);
    g_object_unref (context);
}

#define N_ELEMS 15

static void
gjstest_test_func_gjs_jsapi_util_array(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;
    GjsRootedArray *array;
    int i;
    jsval value;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;

    array = gjs_rooted_array_new();

    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);

    for (i = 0; i < N_ELEMS; i++) {
        value = STRING_TO_JSVAL(JS_NewStringCopyZ(context, "abcdefghijk"));
        gjs_rooted_array_append(context, array, value);
    }

    JS_GC(JS_GetRuntime(context));

    for (i = 0; i < N_ELEMS; i++) {
        char *ascii;

        value = gjs_rooted_array_get(context, array, i);
        g_assert(JSVAL_IS_STRING(value));
        gjs_string_to_utf8(context, value, &ascii);
        g_assert(strcmp(ascii, "abcdefghijk") == 0);
        g_free(ascii);
    }

    gjs_rooted_array_free(context, array, TRUE);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);
}

#undef N_ELEMS

static void
gjstest_test_func_gjs_jsapi_util_string_js_string_utf8(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;

    const char *utf8_string = "\303\211\303\226 foobar \343\203\237";
    char *utf8_result;
    jsval js_string;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;
    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);

    g_assert(gjs_string_from_utf8(context, utf8_string, -1, &js_string) == JS_TRUE);
    g_assert(JSVAL_IS_STRING(js_string));
    g_assert(gjs_string_to_utf8(context, js_string, &utf8_result) == JS_TRUE);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);

    g_assert(g_str_equal(utf8_string, utf8_result));

    g_free(utf8_result);
}

static void
gjstest_test_func_gjs_stack_dump(void)
{
  GjsContext *context;

  /* TODO this test could be better - maybe expose dumpstack as a JS API
   * so that we have a JS stack to dump?  At least here we're getting some
   * coverage.
   */
  context = gjs_context_new();

  gjs_dumpstack();
  g_object_unref(context);
  gjs_dumpstack();
}

static void
gjstest_test_func_gjs_jsapi_util_error_throw(void)
{
    GjsUnitTestFixture fixture;
    JSContext *context;
    JSObject *global;
    jsval exc, value, previous;
    char *s = NULL;
    int strcmp_result;

    _gjs_unit_test_fixture_begin(&fixture);
    context = fixture.context;
    global = JS_GetGlobalObject(context);
    JSCompartment *oldCompartment = JS_EnterCompartment(context, global);
    /* Test that we can throw */

    gjs_throw(context, "This is an exception %d", 42);

    g_assert(JS_IsExceptionPending(context));

    exc = JSVAL_VOID;
    JS_GetPendingException(context, &exc);
    g_assert(!JSVAL_IS_VOID(exc));

    value = JSVAL_VOID;
    JS_GetProperty(context, JSVAL_TO_OBJECT(exc), "message",
                   &value);

    g_assert(JSVAL_IS_STRING(value));

    gjs_string_to_utf8(context, value, &s);
    g_assert(s != NULL);
    strcmp_result = strcmp(s, "This is an exception 42");
    free(s);
    if (strcmp_result != 0)
        g_error("Exception has wrong message '%s'", s);

    /* keep this around before we clear it */
    previous = exc;
    JS_AddValueRoot(context, &previous);

    JS_ClearPendingException(context);

    g_assert(!JS_IsExceptionPending(context));

    /* Check that we don't overwrite a pending exception */
    JS_SetPendingException(context, previous);

    g_assert(JS_IsExceptionPending(context));

    gjs_throw(context, "Second different exception %s", "foo");

    g_assert(JS_IsExceptionPending(context));

    exc = JSVAL_VOID;
    JS_GetPendingException(context, &exc);
    g_assert(!JSVAL_IS_VOID(exc));
    g_assert(JSVAL_TO_OBJECT(exc) == JSVAL_TO_OBJECT(previous));

    JS_RemoveValueRoot(context, &previous);

    JS_LeaveCompartment(context, oldCompartment);
    _gjs_unit_test_fixture_finish(&fixture);
}

static void
gjstest_test_func_util_glib_strv_concat_null(void)
{
    char **ret;

    ret = gjs_g_strv_concat(NULL, 0);
    g_assert(ret != NULL);
    g_assert(ret[0] == NULL);

    g_strfreev(ret);
}

static void
gjstest_test_func_util_glib_strv_concat_pointers(void)
{
    char  *strv0[2] = {(char*)"foo", NULL};
    char  *strv1[1] = {NULL};
    char **strv2    = NULL;
    char  *strv3[2] = {(char*)"bar", NULL};
    char **stuff[4];
    char **ret;

    stuff[0] = strv0;
    stuff[1] = strv1;
    stuff[2] = strv2;
    stuff[3] = strv3;

    ret = gjs_g_strv_concat(stuff, 4);
    g_assert(ret != NULL);
    g_assert_cmpstr(ret[0], ==, strv0[0]);  /* same string */
    g_assert(ret[0] != strv0[0]);           /* different pointer */
    g_assert_cmpstr(ret[1], ==, strv3[0]);
    g_assert(ret[1] != strv3[0]);
    g_assert(ret[2] == NULL);

    g_strfreev(ret);
}

static void
gjstest_test_strip_shebang_no_advance_for_no_shebang(void)
{
    const char *script = "foo\nbar";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert_cmpstr(script, ==, stripped);
    g_assert(script_len == script_len_original);
    g_assert(line_number == 1);
}

static void
gjstest_test_strip_shebang_advance_for_shebang(void)
{
    const char *script = "#!foo\nbar";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert_cmpstr(stripped, ==, "bar");
    g_assert(script_len == 3);
    g_assert(line_number == 2);
}

static void
gjstest_test_strip_shebang_return_null_for_just_shebang(void)
{
    const char *script = "#!foo";
    gssize     script_len_original = strlen(script);
    gssize     script_len = script_len_original;
    int        line_number = 1;

    const char *stripped = gjs_strip_unix_shebang(script,
                                                  &script_len,
                                                  &line_number);

    g_assert(stripped == NULL);
    g_assert(script_len == 0);
    g_assert(line_number == -1);
}

int
main(int    argc,
     char **argv)
{
    gjs_crash_after_timeout(60*7); /* give the unit tests 7 minutes to complete */

    g_test_init(&argc, &argv, NULL);

    g_test_add_func("/gjs/context/construct/destroy", gjstest_test_func_gjs_context_construct_destroy);
    g_test_add_func("/gjs/context/construct/eval", gjstest_test_func_gjs_context_construct_eval);
    g_test_add_func("/gjs/jsapi/util/array", gjstest_test_func_gjs_jsapi_util_array);
    g_test_add_func("/gjs/jsapi/util/error/throw", gjstest_test_func_gjs_jsapi_util_error_throw);
    g_test_add_func("/gjs/jsapi/util/string/js/string/utf8", gjstest_test_func_gjs_jsapi_util_string_js_string_utf8);
    g_test_add_func("/gjs/jsutil/strip_shebang/no_shebang", gjstest_test_strip_shebang_no_advance_for_no_shebang);
    g_test_add_func("/gjs/jsutil/strip_shebang/have_shebang", gjstest_test_strip_shebang_advance_for_shebang);
    g_test_add_func("/gjs/jsutil/strip_shebang/only_shebang", gjstest_test_strip_shebang_return_null_for_just_shebang);
    g_test_add_func("/gjs/stack/dump", gjstest_test_func_gjs_stack_dump);
    g_test_add_func("/util/glib/strv/concat/null", gjstest_test_func_util_glib_strv_concat_null);
    g_test_add_func("/util/glib/strv/concat/pointers", gjstest_test_func_util_glib_strv_concat_pointers);

    g_test_run();

    return 0;
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Galaxy Authors. All Rights Reserved
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// Author: wangtaize@baidu.com

#include "agent/cgroup.h"

#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <errno.h>
#include <boost/bind.hpp>
#include "common/logging.h"
#include "common/util.h"
#include "agent/downloader_manager.h"
namespace galaxy {

static int CPU_CFS_PERIOD = 100000;
static int MIN_CPU_CFS_QUOTA = 1000;

int CGroupCtrl::Create(int64_t task_id, std::map<std::string, std::string>& sub_sys_map) {
    if (_support_cg.size() <= 0) {
        LOG(WARNING, "no subsystem is support");
        return -1;
    }

    std::vector<std::string>::iterator it = _support_cg.begin();

    for (; it != _support_cg.end(); ++it) {
        std::stringstream ss ;
        ss << _cg_root << "/" << *it << "/" << task_id;
        int status = mkdir(ss.str().c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);

        if (status != 0) {
            if (errno == EEXIST) {
                // TODO
                LOG(WARNING, "cgroup already there");
            } else {
                LOG(FATAL, "fail to create subsystem %s ,status is %d", ss.str().c_str(), status);
                return status;
            }
        }

        sub_sys_map[*it] = ss.str();
        LOG(INFO, "create subsystem %s successfully", ss.str().c_str());
    }

    return 0;
}

//目前不支持递归删除
//删除前应该清空tasks文件
int CGroupCtrl::Destroy(int64_t task_id) {
    if (_support_cg.size() <= 0) {
        LOG(WARNING, "no subsystem is support");
        return -1;
    }

    std::vector<std::string>::iterator it = _support_cg.begin();

    for (; it != _support_cg.end(); ++it) {
        std::stringstream ss ;
        ss << _cg_root << "/" << *it << "/" << task_id;
        int status = rmdir(ss.str().c_str());
        if(status != 0 ){
            LOG(FATAL,"fail to delete subsystem %s status %d",ss.str().c_str(),status);
            return status;
        }
    }

    return 0;
}

int AbstractCtrl::AttachTask(pid_t pid) {
    std::string task_file = _my_cg_root + "/" + "tasks";
    int ret = common::util::WriteIntToFile(task_file, pid);

    if (ret < 0) {
        LOG(FATAL, "fail to attach pid  %d for %s", pid, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int MemoryCtrl::SetLimit(int64_t limit) {
    std::string limit_file = _my_cg_root + "/" + "memory.limit_in_bytes";
    int ret = common::util::WriteIntToFile(limit_file, limit);

    if (ret < 0) {
        LOG(FATAL, "fail to set limt %lld for %s", limit, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int MemoryCtrl::SetSoftLimit(int64_t soft_limit) {
    std::string soft_limit_file = _my_cg_root + "/" + "memory.soft_limit_in_bytes";
    int ret = common::util::WriteIntToFile(soft_limit_file, soft_limit);

    if (ret < 0) {
        LOG(FATAL, "fail to set soft limt %lld for %s", soft_limit, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int CpuCtrl::SetCpuShare(int64_t cpu_share) {
    std::string cpu_share_file = _my_cg_root + "/" + "cpu.shares";
    int ret = common::util::WriteIntToFile(cpu_share_file, cpu_share);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu share %lld for %s", cpu_share, _my_cg_root.c_str());
        return -1;
    }

    return 0;

}
int CpuCtrl::SetCpuPeriod(int64_t cpu_period) {
    std::string cpu_period_file = _my_cg_root + "/" + "cpu.cfs_period_us";
    int ret = common::util::WriteIntToFile(cpu_period_file, cpu_period);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu period %lld for %s", cpu_period, _my_cg_root.c_str());
        return -1;
    }

    return 0;

}

int CpuCtrl::SetCpuQuota(int64_t cpu_quota) {
    std::string cpu_quota_file = _my_cg_root + "/" + "cpu.cfs_quota_us";
    int ret = common::util::WriteIntToFile(cpu_quota_file, cpu_quota);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu quota  %ld for %s", cpu_quota, _my_cg_root.c_str());
        return -1;
    }
    LOG(INFO, "set cpu quota %ld for %s", cpu_quota, _my_cg_root.c_str());
    return 0;

}

int ContainerTaskRunner::Prepare() {
    LOG(INFO, "prepare container for task %d", m_task_info.task_id());
    //TODO
    std::vector<std::string> support_cg;
    support_cg.push_back("memory");
    support_cg.push_back("cpu");
    _cg_ctrl = new CGroupCtrl(_cg_root, support_cg);
    std::map<std::string, std::string> sub_sys_map;
    int status = _cg_ctrl->Create(m_task_info.task_id(), sub_sys_map);

    if (status != 0) {
        LOG(FATAL, "fail to create subsystem for task %d,status %d", m_task_info.task_id(), status);
        return status;
    }

    _mem_ctrl = new MemoryCtrl(sub_sys_map["memory"]);
    _cpu_ctrl = new CpuCtrl(sub_sys_map["cpu"]);

    std::string uri = m_task_info.task_raw();
    std::string path = m_workspace->GetPath();
    path.append("/");
    path.append("tmp.tar.gz");

    DownloaderManager* downloader_handler = DownloaderManager::GetInstance();
    downloader_handler->DownloadInThread(
            uri,
            path,
            boost::bind(&ContainerTaskRunner::StartAfterDownload, this, _1));
    return 0;
}

void ContainerTaskRunner::StartAfterDownload(int ret) {
    if (ret == 0) {
        std::string tar_cmd = "cd " + m_workspace->GetPath() + " && tar -xzf tmp.tar.gz";
        int status = system(tar_cmd.c_str());
        if (status != 0) {
            LOG(WARNING, "tar -xf failed");
            return;
        }
        Start();
    }
}

void ContainerTaskRunner::PutToCGroup(){
    int64_t mem_size = m_task_info.required_mem() * (1L << 30);
    double cpu_core = m_task_info.required_cpu();
    LOG(INFO, "resource limit cpu %f, mem %ld", cpu_core, mem_size);
    if (mem_size <= (1L << 30)) {
        mem_size = (1L << 30);
    }
    /*
    std::string mem_key = "memory";
    std::string cpu_key = "cpu";
    for (int i = 0; i< m_task_info.resource_list_size(); i++){
        ResourceItem item = m_task_info.resource_list(i);
        if(mem_key.compare(item.name())==0 && item.value() > 0){
            mem_size = item.value();
            continue;
        }
        if(cpu_key.compare(item.name())==0 && item.value() >0){
            cpu_share = item.value() * 512;
        }

    }*/
    _mem_ctrl->SetLimit(mem_size);
    //_cpu_ctrl->SetCpuShare(cpu_share);
    int64_t quota = static_cast<int64_t>(cpu_core * CPU_CFS_PERIOD);
    if (quota < MIN_CPU_CFS_QUOTA) {
        quota = MIN_CPU_CFS_QUOTA;
    }
    _cpu_ctrl->SetCpuQuota(quota);
    pid_t my_pid = getpid();
    _mem_ctrl->AttachTask(my_pid);
    _cpu_ctrl->AttachTask(my_pid);
}

int ContainerTaskRunner::Start() {
    LOG(INFO, "start a task with id %d", m_task_info.task_id());

    if (IsRunning() == 0) {
        LOG(WARNING, "task with id %d has been runing", m_task_info.task_id());
        return -1;
    }

    int stdout_fd, stderr_fd;
    std::vector<int> fds;
    PrepareStart(fds, &stdout_fd, &stderr_fd);
    m_child_pid = fork();

    if (m_child_pid == 0) {
        PutToCGroup();
        StartTaskAfterFork(fds, stdout_fd, stderr_fd);
    } else {
        close(stdout_fd);
        close(stderr_fd);
    }
    return 0;
}

int ContainerTaskRunner::Stop(){
    int status = AbstractTaskRunner::Stop();
    LOG(INFO,"stop  task %d  with status %d",m_task_info.task_id(),status);
    if(status != 0 ){
        return status;
    }
    status = _cg_ctrl->Destroy(m_task_info.task_id());
    LOG(INFO,"destroy cgroup for task %d with status %s",m_task_info.task_id(),status);
    return status;
}

}
<commit_msg>remove 1<<30<commit_after>// Copyright (c) 2015, Galaxy Authors. All Rights Reserved
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// Author: wangtaize@baidu.com

#include "agent/cgroup.h"

#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
#include <errno.h>
#include <boost/bind.hpp>
#include "common/logging.h"
#include "common/util.h"
#include "agent/downloader_manager.h"
namespace galaxy {

static int CPU_CFS_PERIOD = 100000;
static int MIN_CPU_CFS_QUOTA = 1000;

int CGroupCtrl::Create(int64_t task_id, std::map<std::string, std::string>& sub_sys_map) {
    if (_support_cg.size() <= 0) {
        LOG(WARNING, "no subsystem is support");
        return -1;
    }

    std::vector<std::string>::iterator it = _support_cg.begin();

    for (; it != _support_cg.end(); ++it) {
        std::stringstream ss ;
        ss << _cg_root << "/" << *it << "/" << task_id;
        int status = mkdir(ss.str().c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);

        if (status != 0) {
            if (errno == EEXIST) {
                // TODO
                LOG(WARNING, "cgroup already there");
            } else {
                LOG(FATAL, "fail to create subsystem %s ,status is %d", ss.str().c_str(), status);
                return status;
            }
        }

        sub_sys_map[*it] = ss.str();
        LOG(INFO, "create subsystem %s successfully", ss.str().c_str());
    }

    return 0;
}

//目前不支持递归删除
//删除前应该清空tasks文件
int CGroupCtrl::Destroy(int64_t task_id) {
    if (_support_cg.size() <= 0) {
        LOG(WARNING, "no subsystem is support");
        return -1;
    }

    std::vector<std::string>::iterator it = _support_cg.begin();

    for (; it != _support_cg.end(); ++it) {
        std::stringstream ss ;
        ss << _cg_root << "/" << *it << "/" << task_id;
        int status = rmdir(ss.str().c_str());
        if(status != 0 ){
            LOG(FATAL,"fail to delete subsystem %s status %d",ss.str().c_str(),status);
            return status;
        }
    }

    return 0;
}

int AbstractCtrl::AttachTask(pid_t pid) {
    std::string task_file = _my_cg_root + "/" + "tasks";
    int ret = common::util::WriteIntToFile(task_file, pid);

    if (ret < 0) {
        LOG(FATAL, "fail to attach pid  %d for %s", pid, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int MemoryCtrl::SetLimit(int64_t limit) {
    std::string limit_file = _my_cg_root + "/" + "memory.limit_in_bytes";
    int ret = common::util::WriteIntToFile(limit_file, limit);

    if (ret < 0) {
        LOG(FATAL, "fail to set limt %lld for %s", limit, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int MemoryCtrl::SetSoftLimit(int64_t soft_limit) {
    std::string soft_limit_file = _my_cg_root + "/" + "memory.soft_limit_in_bytes";
    int ret = common::util::WriteIntToFile(soft_limit_file, soft_limit);

    if (ret < 0) {
        LOG(FATAL, "fail to set soft limt %lld for %s", soft_limit, _my_cg_root.c_str());
        return -1;
    }

    return 0;
}

int CpuCtrl::SetCpuShare(int64_t cpu_share) {
    std::string cpu_share_file = _my_cg_root + "/" + "cpu.shares";
    int ret = common::util::WriteIntToFile(cpu_share_file, cpu_share);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu share %lld for %s", cpu_share, _my_cg_root.c_str());
        return -1;
    }

    return 0;

}
int CpuCtrl::SetCpuPeriod(int64_t cpu_period) {
    std::string cpu_period_file = _my_cg_root + "/" + "cpu.cfs_period_us";
    int ret = common::util::WriteIntToFile(cpu_period_file, cpu_period);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu period %lld for %s", cpu_period, _my_cg_root.c_str());
        return -1;
    }

    return 0;

}

int CpuCtrl::SetCpuQuota(int64_t cpu_quota) {
    std::string cpu_quota_file = _my_cg_root + "/" + "cpu.cfs_quota_us";
    int ret = common::util::WriteIntToFile(cpu_quota_file, cpu_quota);
    if (ret < 0) {
        LOG(FATAL, "fail to set cpu quota  %ld for %s", cpu_quota, _my_cg_root.c_str());
        return -1;
    }
    LOG(INFO, "set cpu quota %ld for %s", cpu_quota, _my_cg_root.c_str());
    return 0;

}

int ContainerTaskRunner::Prepare() {
    LOG(INFO, "prepare container for task %d", m_task_info.task_id());
    //TODO
    std::vector<std::string> support_cg;
    support_cg.push_back("memory");
    support_cg.push_back("cpu");
    _cg_ctrl = new CGroupCtrl(_cg_root, support_cg);
    std::map<std::string, std::string> sub_sys_map;
    int status = _cg_ctrl->Create(m_task_info.task_id(), sub_sys_map);

    if (status != 0) {
        LOG(FATAL, "fail to create subsystem for task %d,status %d", m_task_info.task_id(), status);
        return status;
    }

    _mem_ctrl = new MemoryCtrl(sub_sys_map["memory"]);
    _cpu_ctrl = new CpuCtrl(sub_sys_map["cpu"]);

    std::string uri = m_task_info.task_raw();
    std::string path = m_workspace->GetPath();
    path.append("/");
    path.append("tmp.tar.gz");

    DownloaderManager* downloader_handler = DownloaderManager::GetInstance();
    downloader_handler->DownloadInThread(
            uri,
            path,
            boost::bind(&ContainerTaskRunner::StartAfterDownload, this, _1));
    return 0;
}

void ContainerTaskRunner::StartAfterDownload(int ret) {
    if (ret == 0) {
        std::string tar_cmd = "cd " + m_workspace->GetPath() + " && tar -xzf tmp.tar.gz";
        int status = system(tar_cmd.c_str());
        if (status != 0) {
            LOG(WARNING, "tar -xf failed");
            return;
        }
        Start();
    }
}

void ContainerTaskRunner::PutToCGroup(){
    int64_t mem_size = m_task_info.required_mem();
    double cpu_core = m_task_info.required_cpu();
    LOG(INFO, "resource limit cpu %f, mem %ld", cpu_core, mem_size);
    /*
    std::string mem_key = "memory";
    std::string cpu_key = "cpu";
    for (int i = 0; i< m_task_info.resource_list_size(); i++){
        ResourceItem item = m_task_info.resource_list(i);
        if(mem_key.compare(item.name())==0 && item.value() > 0){
            mem_size = item.value();
            continue;
        }
        if(cpu_key.compare(item.name())==0 && item.value() >0){
            cpu_share = item.value() * 512;
        }

    }*/
    _mem_ctrl->SetLimit(mem_size);
    //_cpu_ctrl->SetCpuShare(cpu_share);
    int64_t quota = static_cast<int64_t>(cpu_core * CPU_CFS_PERIOD);
    if (quota < MIN_CPU_CFS_QUOTA) {
        quota = MIN_CPU_CFS_QUOTA;
    }
    _cpu_ctrl->SetCpuQuota(quota);
    pid_t my_pid = getpid();
    _mem_ctrl->AttachTask(my_pid);
    _cpu_ctrl->AttachTask(my_pid);
}

int ContainerTaskRunner::Start() {
    LOG(INFO, "start a task with id %d", m_task_info.task_id());

    if (IsRunning() == 0) {
        LOG(WARNING, "task with id %d has been runing", m_task_info.task_id());
        return -1;
    }

    int stdout_fd, stderr_fd;
    std::vector<int> fds;
    PrepareStart(fds, &stdout_fd, &stderr_fd);
    m_child_pid = fork();

    if (m_child_pid == 0) {
        PutToCGroup();
        StartTaskAfterFork(fds, stdout_fd, stderr_fd);
    } else {
        close(stdout_fd);
        close(stderr_fd);
    }
    return 0;
}

int ContainerTaskRunner::Stop(){
    int status = AbstractTaskRunner::Stop();
    LOG(INFO,"stop  task %d  with status %d",m_task_info.task_id(),status);
    if(status != 0 ){
        return status;
    }
    status = _cg_ctrl->Destroy(m_task_info.task_id());
    LOG(INFO,"destroy cgroup for task %d with status %s",m_task_info.task_id(),status);
    return status;
}

}
<|endoftext|>


<commit_before>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/kernels/captured_function.h"

#include <utility>

#include "tensorflow/core/common_runtime/threadpool_device.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/lookup_interface.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/queue_interface.h"
#include "tensorflow/core/framework/reader_interface.h"
#include "tensorflow/core/framework/resource_handle.pb_text.h"
#include "tensorflow/core/kernels/dataset.h"
#include "tensorflow/core/kernels/variable_ops.h"
#include "tensorflow/core/platform/notification.h"
#include "tensorflow/core/public/session_options.h"

namespace tensorflow {

/* static */
Status CapturedFunction::Create(
    OpKernelContext* ctx, const NameAttrList* func, int graph_def_version,
    std::vector<Tensor> captured_inputs,
    std::unique_ptr<CapturedFunction>* out_function) {
  // NOTE(mrry): We need to assign a name to the device, and we choose
  // the same name as the calling context's device so that we do not
  // need to rewrite resource handles that are found in `captured_inputs`.
  Device* device =
      new ThreadPoolDevice(SessionOptions(), ctx->device()->attributes().name(),
                           Bytes(256 << 20), DeviceLocality(), cpu_allocator());

// TODO(mrry): Handle arbitrary resource types, which might require a
// redesign (or opening up access to `ResourceMgr::DoLookup()` and
// `ResourceMgr::DoCreate()` to this code).
#define HANDLE_RESOURCE_TYPE(ResourceType)                                     \
  if (input_handle.hash_code() == MakeTypeIndex<ResourceType>().hash_code()) { \
    ResourceType* resource;                                                    \
    Status s = LookupResource(ctx, input_handle, &resource);                   \
    if (errors::IsNotFound(s)) {                                               \
      return errors::FailedPrecondition(                                       \
          "Failed to capture resource named \"", input_handle.name(),          \
          "\" in a dataset function. You may need to initialize it "           \
          "explicitly before initializing an iterator that uses it.");         \
    } else if (!s.ok()) {                                                      \
      return s;                                                                \
    }                                                                          \
    TF_RETURN_IF_ERROR(device->resource_manager()->Create(                     \
        input_handle.container(), input_handle.name(), resource));             \
    continue;                                                                  \
  }

  for (size_t i = 0; i < captured_inputs.size(); ++i) {
    if (captured_inputs[i].dtype() == DT_RESOURCE) {
      // Extract the resource from `ctx->resource_manager()` and
      // insert it into `device->resource_manager()` so that it can be
      // used when the function executes.
      ResourceHandle input_handle =
          captured_inputs[i].scalar<ResourceHandle>()();
      HANDLE_RESOURCE_TYPE(lookup::LookupInterface);
      HANDLE_RESOURCE_TYPE(QueueInterface);
      HANDLE_RESOURCE_TYPE(Var);
      return errors::Unimplemented(
          "Cannot currently capture resource '",
          ProtoDebugString(input_handle),
          "' in a dataset function (type not supported).");
    }
  }
#undef HANDLE_RESOURCE_TYPE

  std::unique_ptr<DeviceMgr> device_mgr(new DeviceMgr({device}));
  std::unique_ptr<FunctionLibraryDefinition> flib_def(
      new FunctionLibraryDefinition(
          *ctx->function_library()->GetFunctionLibraryDefinition()));
  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(
      new ProcessFunctionLibraryRuntime(
          device_mgr.get(), ctx->env(), graph_def_version, flib_def.get(),
          {} /* TODO(mrry): OptimizerOptions? */));

  FunctionLibraryRuntime* lib = pflr->GetFLR(device->name());

  FunctionLibraryRuntime::Handle f_handle;
  TF_RETURN_IF_ERROR(
      lib->Instantiate(func->name(), AttrSlice(&func->attr()), &f_handle));

  out_function->reset(new CapturedFunction(
      device, std::move(device_mgr), std::move(flib_def), std::move(pflr), lib,
      f_handle, std::move(captured_inputs)));
  return Status::OK();
}

Status CapturedFunction::Run(FunctionLibraryRuntime::Options f_opts,
                             gtl::ArraySlice<Tensor> args,
                             std::vector<Tensor>* rets, const string& prefix) {
  port::Tracing::TraceMe activity(strings::StrCat(prefix, "::Run"));
  Notification n;
  Status s;
  auto done_callback = [&n, &s](Status func_status) {
    s.Update(func_status);
    n.Notify();
  };
  // TODO(mrry): Add cancellation manager support to IteratorContext
  // so that we can cancel running map functions. The local
  // cancellation manager here is created so that we can run kernels
  // (such as queue kernels) that depend on the non-nullness
  // `OpKernelContext::cancellation_manager()`, but additional effort
  // will be required to plumb it through the `IteratorContext`.
  CancellationManager c_mgr;
  f_opts.cancellation_manager = &c_mgr;
  // TODO(mrry): Implement a synchronous version of
  // FunctionLibraryRuntime::Run() that avoids a context switch for small
  // functions.
  if (captured_inputs_.empty()) {
    lib_->Run(f_opts, f_handle_, args, rets, done_callback);
  } else {
    std::vector<Tensor> args_with_captured;
    args_with_captured.reserve(args.size() + captured_inputs_.size());
    args_with_captured.insert(args_with_captured.end(), args.begin(),
                              args.end());
    args_with_captured.insert(args_with_captured.end(),
                              captured_inputs_.begin(), captured_inputs_.end());
    lib_->Run(f_opts, f_handle_, args_with_captured, rets, done_callback);
  }
  n.WaitForNotification();
  return s;
}

CapturedFunction::CapturedFunction(
    Device* device, std::unique_ptr<DeviceMgr> device_mgr,
    std::unique_ptr<FunctionLibraryDefinition> flib_def,
    std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
    FunctionLibraryRuntime* lib, FunctionLibraryRuntime::Handle f_handle,
    std::vector<Tensor> captured_inputs)
    : device_(device),
      device_mgr_(std::move(device_mgr)),
      flib_def_(std::move(flib_def)),
      pflr_(std::move(pflr)),
      lib_(lib),
      f_handle_(f_handle),
      captured_inputs_(std::move(captured_inputs)) {}

}  // namespace tensorflow
<commit_msg>Use 2-arg TraceMe constructor to prevent unnecessary StrCat computation when tracing is disabled.<commit_after>/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/kernels/captured_function.h"

#include <utility>

#include "tensorflow/core/common_runtime/threadpool_device.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/lookup_interface.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/queue_interface.h"
#include "tensorflow/core/framework/reader_interface.h"
#include "tensorflow/core/framework/resource_handle.pb_text.h"
#include "tensorflow/core/kernels/dataset.h"
#include "tensorflow/core/kernels/variable_ops.h"
#include "tensorflow/core/platform/notification.h"
#include "tensorflow/core/public/session_options.h"

namespace tensorflow {

/* static */
Status CapturedFunction::Create(
    OpKernelContext* ctx, const NameAttrList* func, int graph_def_version,
    std::vector<Tensor> captured_inputs,
    std::unique_ptr<CapturedFunction>* out_function) {
  // NOTE(mrry): We need to assign a name to the device, and we choose
  // the same name as the calling context's device so that we do not
  // need to rewrite resource handles that are found in `captured_inputs`.
  Device* device =
      new ThreadPoolDevice(SessionOptions(), ctx->device()->attributes().name(),
                           Bytes(256 << 20), DeviceLocality(), cpu_allocator());

// TODO(mrry): Handle arbitrary resource types, which might require a
// redesign (or opening up access to `ResourceMgr::DoLookup()` and
// `ResourceMgr::DoCreate()` to this code).
#define HANDLE_RESOURCE_TYPE(ResourceType)                                     \
  if (input_handle.hash_code() == MakeTypeIndex<ResourceType>().hash_code()) { \
    ResourceType* resource;                                                    \
    Status s = LookupResource(ctx, input_handle, &resource);                   \
    if (errors::IsNotFound(s)) {                                               \
      return errors::FailedPrecondition(                                       \
          "Failed to capture resource named \"", input_handle.name(),          \
          "\" in a dataset function. You may need to initialize it "           \
          "explicitly before initializing an iterator that uses it.");         \
    } else if (!s.ok()) {                                                      \
      return s;                                                                \
    }                                                                          \
    TF_RETURN_IF_ERROR(device->resource_manager()->Create(                     \
        input_handle.container(), input_handle.name(), resource));             \
    continue;                                                                  \
  }

  for (size_t i = 0; i < captured_inputs.size(); ++i) {
    if (captured_inputs[i].dtype() == DT_RESOURCE) {
      // Extract the resource from `ctx->resource_manager()` and
      // insert it into `device->resource_manager()` so that it can be
      // used when the function executes.
      ResourceHandle input_handle =
          captured_inputs[i].scalar<ResourceHandle>()();
      HANDLE_RESOURCE_TYPE(lookup::LookupInterface);
      HANDLE_RESOURCE_TYPE(QueueInterface);
      HANDLE_RESOURCE_TYPE(Var);
      return errors::Unimplemented(
          "Cannot currently capture resource '",
          ProtoDebugString(input_handle),
          "' in a dataset function (type not supported).");
    }
  }
#undef HANDLE_RESOURCE_TYPE

  std::unique_ptr<DeviceMgr> device_mgr(new DeviceMgr({device}));
  std::unique_ptr<FunctionLibraryDefinition> flib_def(
      new FunctionLibraryDefinition(
          *ctx->function_library()->GetFunctionLibraryDefinition()));
  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(
      new ProcessFunctionLibraryRuntime(
          device_mgr.get(), ctx->env(), graph_def_version, flib_def.get(),
          {} /* TODO(mrry): OptimizerOptions? */));

  FunctionLibraryRuntime* lib = pflr->GetFLR(device->name());

  FunctionLibraryRuntime::Handle f_handle;
  TF_RETURN_IF_ERROR(
      lib->Instantiate(func->name(), AttrSlice(&func->attr()), &f_handle));

  out_function->reset(new CapturedFunction(
      device, std::move(device_mgr), std::move(flib_def), std::move(pflr), lib,
      f_handle, std::move(captured_inputs)));
  return Status::OK();
}

Status CapturedFunction::Run(FunctionLibraryRuntime::Options f_opts,
                             gtl::ArraySlice<Tensor> args,
                             std::vector<Tensor>* rets, const string& prefix) {
  port::Tracing::TraceMe activity(prefix, "::Run");
  Notification n;
  Status s;
  auto done_callback = [&n, &s](Status func_status) {
    s.Update(func_status);
    n.Notify();
  };
  // TODO(mrry): Add cancellation manager support to IteratorContext
  // so that we can cancel running map functions. The local
  // cancellation manager here is created so that we can run kernels
  // (such as queue kernels) that depend on the non-nullness
  // `OpKernelContext::cancellation_manager()`, but additional effort
  // will be required to plumb it through the `IteratorContext`.
  CancellationManager c_mgr;
  f_opts.cancellation_manager = &c_mgr;
  // TODO(mrry): Implement a synchronous version of
  // FunctionLibraryRuntime::Run() that avoids a context switch for small
  // functions.
  if (captured_inputs_.empty()) {
    lib_->Run(f_opts, f_handle_, args, rets, done_callback);
  } else {
    std::vector<Tensor> args_with_captured;
    args_with_captured.reserve(args.size() + captured_inputs_.size());
    args_with_captured.insert(args_with_captured.end(), args.begin(),
                              args.end());
    args_with_captured.insert(args_with_captured.end(),
                              captured_inputs_.begin(), captured_inputs_.end());
    lib_->Run(f_opts, f_handle_, args_with_captured, rets, done_callback);
  }
  n.WaitForNotification();
  return s;
}

CapturedFunction::CapturedFunction(
    Device* device, std::unique_ptr<DeviceMgr> device_mgr,
    std::unique_ptr<FunctionLibraryDefinition> flib_def,
    std::unique_ptr<ProcessFunctionLibraryRuntime> pflr,
    FunctionLibraryRuntime* lib, FunctionLibraryRuntime::Handle f_handle,
    std::vector<Tensor> captured_inputs)
    : device_(device),
      device_mgr_(std::move(device_mgr)),
      flib_def_(std::move(flib_def)),
      pflr_(std::move(pflr)),
      lib_(lib),
      f_handle_(f_handle),
      captured_inputs_(std::move(captured_inputs)) {}

}  // namespace tensorflow
<|endoftext|>