<commit_before>/*
  Copyright (C) 2006 by Marten Svanfeldt

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Library General Public
  License as published by the Free Software Foundation; either
  version 2 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Library General Public License for more details.

  You should have received a copy of the GNU Library General Public
  License along with this library; if not, write to the Free
  Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "common.h"

#include "config.h"
#include "statistics.h"
#include "lighter.h"
#include "tui.h"

namespace lighter
{
  TUI globalTUI;

  TUI::TUI ()
    : scfImplementationType (this), messageBufferEnd (0),
    simpleMode (false), prevWasReporter (false),
    kdLastNumNodes (0), lastTaskProgress (0.0f)
  {
  }

  void TUI::Initialize (iObjectRegistry* objReg)
  {
    object_reg = objReg;
    simpleMode = globalLighter->configMgr->GetBool ("lighter2.simpletui", false);
  }

  void TUI::Redraw (int drawFlags)
  {
    // Redraw the "static" background
    if (simpleMode)
    {
      DrawSimple ();
      return;
    }

    // Clear
    if (drawFlags & TUI_DRAW_CLEAR)
      csPrintf (CS_ANSI_CLEAR_SCREEN);

    // Screen layout (keep to 80x25...)
    if (drawFlags & TUI_DRAW_STATIC)
      DrawStatic ();

    // Draw progress
    if (drawFlags & TUI_DRAW_PROGRESS)
      DrawProgress ();

    // Draw messages
    if (drawFlags & TUI_DRAW_MESSAGES)
      DrawMessage ();

    // Draw RayCore stats
    if (drawFlags & TUI_DRAW_RAYCORE)
      DrawRayCore ();

    // Draw global settings
    if (drawFlags & TUI_DRAW_SETTINGS)
      DrawSettings ();

    // Draw global stats
    if (drawFlags & TUI_DRAW_STATS)
      DrawStats ();

    /* Linux: output is buffered, and the UI may appear "incomplete" if not
     * flushed */
    fflush (stdout);
  }

  void TUI::FinishDraw ()
  {
    if (simpleMode)
    {
      DrawSimpleEnd ();
    }
  }

  static const char* TUI_SEVERITY_TEXT[] =
  {
    CS_ANSI_FK CS_ANSI_BW "B" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BR "X" CS_ANSI_RST " ",
    CS_ANSI_FK CS_ANSI_BY "!" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BB "i" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BW "D" CS_ANSI_RST " "
  };

  THREADED_CALLABLE_IMPL4(TUI, Report, iReporter* reporter, int severity,
    const char* msgId, const char* description)
  {
    csStringArray descrSplit;
    descrSplit.SplitString (description, "\n");

    for (size_t i = descrSplit.GetSize(); i-- > 0;)
    {
      csString& buf = messageBuffer[messageBufferEnd];
      buf.Clear ();

      buf.Append (TUI_SEVERITY_TEXT[severity]);
      csString tmp(descrSplit[i]);
      buf.Append (tmp.Slice (0,74).PadRight (74));

      if (simpleMode)
      {
        if (!prevWasReporter)
          csPrintf ("\n");
        csPrintf ("%s\n", buf.GetDataSafe ());
        prevWasReporter = true;
      }

      messageBufferEnd++;
      if(messageBufferEnd > 3) messageBufferEnd = 0;
    }

    // Print them
    Redraw (TUI::TUI_DRAW_MESSAGES);

    // Also hand it off to the standard listener
    return false;
  }

  void TUI::DrawStatic () const
  {
    csPrintf ("/------------------------    Crystal Space Lighter    ------------------------\\\n");
    csPrintf ("| Total progress:                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("| Part progress:                                                              |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|-----------------------------------------------------------------------------|\n");
    csPrintf ("| Rays:    | Settings   | Scene Stats                                         |\n");
    csPrintf ("|          | [ ] DL     | S:                                                  |\n");
    csPrintf ("|          | [ ] GI     | O:                                                  |\n");
    csPrintf ("|          | [ ] LMs    | L:                                                  |\n");
    csPrintf ("|          | [ ] AO     | LM:                                                 |\n");
    csPrintf ("|          | PLM:       |                                                     |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|          | ALM:       | KD-stats                                            |\n");
    csPrintf ("|          |            | N:                                                  |\n");
    csPrintf ("|          | Density:   | D:                                                  |\n");
    csPrintf ("|          |            | P:                                                  |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|- CS Messages ---------------------------------------------------------------|\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("\\-----------------------------------------------------------------------------/\n");
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawStats () const
  {
    csPrintf (CS_ANSI_CURSOR(30,15) "%8zu / %8zu", globalStats.kdtree.numNodes, globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(30,16) "%8zu / %8.03f", globalStats.kdtree.maxDepth,
      (float)globalStats.kdtree.sumDepth / (float)globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(30,17) "%8zu / %8.03f", globalStats.kdtree.numPrimitives,
      (float)globalStats.kdtree.numPrimitives / (float)globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawSettings () const
  {
    csPrintf (CS_ANSI_CURSOR(15,8) "%s", globalConfig.GetLighterProperties ().doDirectLight ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,9) "%s", false ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,10) "%s", true ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,11) "%s", false ? "X" : "");

    csPrintf (CS_ANSI_CURSOR(14,13) "%#4.2g", globalConfig.GetDIProperties ().pointLightMultiplier);
    csPrintf (CS_ANSI_CURSOR(14,15) "%#4.2g", globalConfig.GetDIProperties ().areaLightMultiplier);

    csPrintf (CS_ANSI_CURSOR(14,17) "%#4.2g", globalConfig.GetLMProperties ().lmDensity);

    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawRayCore () const
  {
    // Rays
    const char* siConv[] = {" ", "K", "M", "G", "T"};

    uint64 rays = globalStats.raytracer.numRays;
    int prefix = 0;

    while (rays > CONST_UINT64(99999) && prefix < 5)
    {
      rays /= CONST_UINT64(1000);
      prefix++;
    }

    csPrintf (CS_ANSI_CURSOR(3,8) "%6" PRIu64 " %s", rays, siConv[prefix]);
  }

  void TUI::DrawProgress () const
  {
    const uint overallProg = globalStats.progress.GetOverallProgress();
    const uint taskProg = globalStats.progress.GetTaskProgress();

    csString taskName = globalStats.progress.GetTaskName();
    static const size_t maxTaskNameLen = 40;
    if (taskName.Length() > maxTaskNameLen)
      taskName = taskName.Slice (0, maxTaskNameLen-3) + "...";

    csPrintf (CS_ANSI_CURSOR(71, 2));
    csPrintf ("%3d%%", overallProg);

    csPrintf (CS_ANSI_CURSOR(3, 3));
    csPrintf ("%s", GetProgressBar (overallProg).GetDataSafe ());

    csPrintf (CS_ANSI_CURSOR(18, 4));
    csPrintf ("%s", taskName.GetDataSafe ());

    csPrintf (CS_ANSI_CURSOR(71, 4));
    csPrintf ("%3d%%", taskProg);

    csPrintf (CS_ANSI_CURSOR(3, 5));
    csPrintf ("%s", GetProgressBar (taskProg).GetDataSafe ());
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawMessage () const
  {
    csPrintf (CS_ANSI_CURSOR(3, 21));

    // Draw the four buffers, starting with messageBufferEnd
    int row = messageBufferEnd-1;
    if(row <  0) row = 3;

    for(uint i = 0; i < 4; i++)
    {
      csPrintf ("%s", messageBuffer[row].GetDataSafe ());

      row--;
      if(row < 0) row = 3;
      csPrintf (CS_ANSI_CURSOR_DOWN(1));
      csPrintf (CS_ANSI_CURSOR_BWD(100) CS_ANSI_CURSOR_FWD(2));
    }
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  csString TUI::GetProgressBar (uint percent) const
  {
    csString result;

    percent = csClamp(percent, 100u, 0u);

    result.SetCapacity (73);
    //Draw a progess bar being in total 73+2 characters wide
    //|========== =--------- ---------- ---------- ---------- ---------- ---------- ---|

    result.Append ('|');

    uint numDone = (73*percent) / 100;
    uint numToDo = (73*(100-percent)) / 100;

    if(numDone + numToDo < 73) numToDo++;

    for(uint i = 0; i < numDone; ++i)
    {
      result.Append ('=');
    }

    for(uint i = 0; i < numToDo; ++i)
    {
      result.Append ('-');
    }

    result.Append ('|');

    return result;
  }

  void TUI::DrawSimple ()
  {
    const char* lt = (const char*)lastTask;
    const char* tn = globalStats.progress.GetTaskName ();
    if ((lt == 0 && tn != 0) || (lt != 0 && lastTask != tn))
    {
      lastTask = globalStats.progress.GetTaskName();

      prevWasReporter = false;
      csPrintf ("\n% 4d %% - %s ",
        globalStats.progress.GetOverallProgress(),
        lastTask.GetDataSafe());

      // Print new task and global progress
      lastTaskProgress = 0;
    }
    else
    {
      while (lastTaskProgress + 10 < globalStats.progress.GetTaskProgress())
      {
        prevWasReporter = false;
        csPrintf (".");
        lastTaskProgress += 10;
      }
    }
  }

  void TUI::DrawSimpleEnd ()
  {
    // Print KD-tree stats
    csPrintf ("\nKD-tree: \n");
    csPrintf ("N: %8zu / %8zu\n", globalStats.kdtree.numNodes, globalStats.kdtree.leafNodes);
    csPrintf ("D: %8zu / %8.03f\n", globalStats.kdtree.maxDepth,
      (float)globalStats.kdtree.sumDepth / (float)globalStats.kdtree.leafNodes);
    csPrintf ("P: %8zu / %8.03f\n", globalStats.kdtree.numPrimitives,
      (float)globalStats.kdtree.numPrimitives / (float)globalStats.kdtree.leafNodes);

    kdLastNumNodes = globalStats.kdtree.numNodes;
  }

}
<commit_msg>Flush stdout after simpletui updates. Improves output on Linux<commit_after>/*
  Copyright (C) 2006 by Marten Svanfeldt

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Library General Public
  License as published by the Free Software Foundation; either
  version 2 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Library General Public License for more details.

  You should have received a copy of the GNU Library General Public
  License along with this library; if not, write to the Free
  Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/

#include "common.h"

#include "config.h"
#include "statistics.h"
#include "lighter.h"
#include "tui.h"

namespace lighter
{
  TUI globalTUI;

  TUI::TUI ()
    : scfImplementationType (this), messageBufferEnd (0),
    simpleMode (false), prevWasReporter (false),
    kdLastNumNodes (0), lastTaskProgress (0.0f)
  {
  }

  void TUI::Initialize (iObjectRegistry* objReg)
  {
    object_reg = objReg;
    simpleMode = globalLighter->configMgr->GetBool ("lighter2.simpletui", false);
  }

  void TUI::Redraw (int drawFlags)
  {
    // Redraw the "static" background
    if (simpleMode)
    {
      DrawSimple ();
      return;
    }

    // Clear
    if (drawFlags & TUI_DRAW_CLEAR)
      csPrintf (CS_ANSI_CLEAR_SCREEN);

    // Screen layout (keep to 80x25...)
    if (drawFlags & TUI_DRAW_STATIC)
      DrawStatic ();

    // Draw progress
    if (drawFlags & TUI_DRAW_PROGRESS)
      DrawProgress ();

    // Draw messages
    if (drawFlags & TUI_DRAW_MESSAGES)
      DrawMessage ();

    // Draw RayCore stats
    if (drawFlags & TUI_DRAW_RAYCORE)
      DrawRayCore ();

    // Draw global settings
    if (drawFlags & TUI_DRAW_SETTINGS)
      DrawSettings ();

    // Draw global stats
    if (drawFlags & TUI_DRAW_STATS)
      DrawStats ();

    /* Linux: output is buffered, and the UI may appear "incomplete" if not
     * flushed */
    fflush (stdout);
  }

  void TUI::FinishDraw ()
  {
    if (simpleMode)
    {
      DrawSimpleEnd ();
    }
  }

  static const char* TUI_SEVERITY_TEXT[] =
  {
    CS_ANSI_FK CS_ANSI_BW "B" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BR "X" CS_ANSI_RST " ",
    CS_ANSI_FK CS_ANSI_BY "!" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BB "i" CS_ANSI_RST " ",
    CS_ANSI_FW CS_ANSI_BW "D" CS_ANSI_RST " "
  };

  THREADED_CALLABLE_IMPL4(TUI, Report, iReporter* reporter, int severity,
    const char* msgId, const char* description)
  {
    csStringArray descrSplit;
    descrSplit.SplitString (description, "\n");

    for (size_t i = descrSplit.GetSize(); i-- > 0;)
    {
      csString& buf = messageBuffer[messageBufferEnd];
      buf.Clear ();

      buf.Append (TUI_SEVERITY_TEXT[severity]);
      csString tmp(descrSplit[i]);
      buf.Append (tmp.Slice (0,74).PadRight (74));

      if (simpleMode)
      {
        if (!prevWasReporter)
          csPrintf ("\n");
        csPrintf ("%s\n", buf.GetDataSafe ());
        prevWasReporter = true;
      }

      messageBufferEnd++;
      if(messageBufferEnd > 3) messageBufferEnd = 0;
    }

    // Print them
    Redraw (TUI::TUI_DRAW_MESSAGES);

    // Also hand it off to the standard listener
    return false;
  }

  void TUI::DrawStatic () const
  {
    csPrintf ("/------------------------    Crystal Space Lighter    ------------------------\\\n");
    csPrintf ("| Total progress:                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("| Part progress:                                                              |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|-----------------------------------------------------------------------------|\n");
    csPrintf ("| Rays:    | Settings   | Scene Stats                                         |\n");
    csPrintf ("|          | [ ] DL     | S:                                                  |\n");
    csPrintf ("|          | [ ] GI     | O:                                                  |\n");
    csPrintf ("|          | [ ] LMs    | L:                                                  |\n");
    csPrintf ("|          | [ ] AO     | LM:                                                 |\n");
    csPrintf ("|          | PLM:       |                                                     |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|          | ALM:       | KD-stats                                            |\n");
    csPrintf ("|          |            | N:                                                  |\n");
    csPrintf ("|          | Density:   | D:                                                  |\n");
    csPrintf ("|          |            | P:                                                  |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|          |            |                                                     |\n");
    csPrintf ("|- CS Messages ---------------------------------------------------------------|\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("|                                                                             |\n");
    csPrintf ("\\-----------------------------------------------------------------------------/\n");
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawStats () const
  {
    csPrintf (CS_ANSI_CURSOR(30,15) "%8zu / %8zu", globalStats.kdtree.numNodes, globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(30,16) "%8zu / %8.03f", globalStats.kdtree.maxDepth,
      (float)globalStats.kdtree.sumDepth / (float)globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(30,17) "%8zu / %8.03f", globalStats.kdtree.numPrimitives,
      (float)globalStats.kdtree.numPrimitives / (float)globalStats.kdtree.leafNodes);
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawSettings () const
  {
    csPrintf (CS_ANSI_CURSOR(15,8) "%s", globalConfig.GetLighterProperties ().doDirectLight ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,9) "%s", false ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,10) "%s", true ? "X" : "");
    csPrintf (CS_ANSI_CURSOR(15,11) "%s", false ? "X" : "");

    csPrintf (CS_ANSI_CURSOR(14,13) "%#4.2g", globalConfig.GetDIProperties ().pointLightMultiplier);
    csPrintf (CS_ANSI_CURSOR(14,15) "%#4.2g", globalConfig.GetDIProperties ().areaLightMultiplier);

    csPrintf (CS_ANSI_CURSOR(14,17) "%#4.2g", globalConfig.GetLMProperties ().lmDensity);

    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawRayCore () const
  {
    // Rays
    const char* siConv[] = {" ", "K", "M", "G", "T"};

    uint64 rays = globalStats.raytracer.numRays;
    int prefix = 0;

    while (rays > CONST_UINT64(99999) && prefix < 5)
    {
      rays /= CONST_UINT64(1000);
      prefix++;
    }

    csPrintf (CS_ANSI_CURSOR(3,8) "%6" PRIu64 " %s", rays, siConv[prefix]);
  }

  void TUI::DrawProgress () const
  {
    const uint overallProg = globalStats.progress.GetOverallProgress();
    const uint taskProg = globalStats.progress.GetTaskProgress();

    csString taskName = globalStats.progress.GetTaskName();
    static const size_t maxTaskNameLen = 40;
    if (taskName.Length() > maxTaskNameLen)
      taskName = taskName.Slice (0, maxTaskNameLen-3) + "...";

    csPrintf (CS_ANSI_CURSOR(71, 2));
    csPrintf ("%3d%%", overallProg);

    csPrintf (CS_ANSI_CURSOR(3, 3));
    csPrintf ("%s", GetProgressBar (overallProg).GetDataSafe ());

    csPrintf (CS_ANSI_CURSOR(18, 4));
    csPrintf ("%s", taskName.GetDataSafe ());

    csPrintf (CS_ANSI_CURSOR(71, 4));
    csPrintf ("%3d%%", taskProg);

    csPrintf (CS_ANSI_CURSOR(3, 5));
    csPrintf ("%s", GetProgressBar (taskProg).GetDataSafe ());
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  void TUI::DrawMessage () const
  {
    csPrintf (CS_ANSI_CURSOR(3, 21));

    // Draw the four buffers, starting with messageBufferEnd
    int row = messageBufferEnd-1;
    if(row <  0) row = 3;

    for(uint i = 0; i < 4; i++)
    {
      csPrintf ("%s", messageBuffer[row].GetDataSafe ());

      row--;
      if(row < 0) row = 3;
      csPrintf (CS_ANSI_CURSOR_DOWN(1));
      csPrintf (CS_ANSI_CURSOR_BWD(100) CS_ANSI_CURSOR_FWD(2));
    }
    csPrintf (CS_ANSI_CURSOR(1,1));
  }

  csString TUI::GetProgressBar (uint percent) const
  {
    csString result;

    percent = csClamp(percent, 100u, 0u);

    result.SetCapacity (73);
    //Draw a progess bar being in total 73+2 characters wide
    //|========== =--------- ---------- ---------- ---------- ---------- ---------- ---|

    result.Append ('|');

    uint numDone = (73*percent) / 100;
    uint numToDo = (73*(100-percent)) / 100;

    if(numDone + numToDo < 73) numToDo++;

    for(uint i = 0; i < numDone; ++i)
    {
      result.Append ('=');
    }

    for(uint i = 0; i < numToDo; ++i)
    {
      result.Append ('-');
    }

    result.Append ('|');

    return result;
  }

  void TUI::DrawSimple ()
  {
    bool doFlush = false;
    const char* lt = (const char*)lastTask;
    const char* tn = globalStats.progress.GetTaskName ();
    if ((lt == 0 && tn != 0) || (lt != 0 && lastTask != tn))
    {
      lastTask = globalStats.progress.GetTaskName();

      prevWasReporter = false;
      csPrintf ("\n% 4d %% - %s ",
        globalStats.progress.GetOverallProgress(),
        lastTask.GetDataSafe());
      doFlush = true;

      // Print new task and global progress
      lastTaskProgress = 0;
    }
    else
    {
      while (lastTaskProgress + 10 < globalStats.progress.GetTaskProgress())
      {
        prevWasReporter = false;
        csPrintf (".");
        lastTaskProgress += 10;
        doFlush = true;
      }
    }
    if (doFlush) fflush (stdout);
  }

  void TUI::DrawSimpleEnd ()
  {
    // Print KD-tree stats
    csPrintf ("\nKD-tree: \n");
    csPrintf ("N: %8zu / %8zu\n", globalStats.kdtree.numNodes, globalStats.kdtree.leafNodes);
    csPrintf ("D: %8zu / %8.03f\n", globalStats.kdtree.maxDepth,
      (float)globalStats.kdtree.sumDepth / (float)globalStats.kdtree.leafNodes);
    csPrintf ("P: %8zu / %8.03f\n", globalStats.kdtree.numPrimitives,
      (float)globalStats.kdtree.numPrimitives / (float)globalStats.kdtree.leafNodes);

    kdLastNumNodes = globalStats.kdtree.numNodes;
  }

}
<|endoftext|>


<commit_before>#include <TMath.h>
#include <TGraph2D.h>
#include <TRandom.h>
#include <TStyle.h>
#include <TCanvas.h>
#include <TF2.h>

void graph2dfit()
{
   gStyle->SetOptStat(0);
   gStyle->SetOptFit();
   gStyle->SetPadBorderMode(0);
   gStyle->SetFrameBorderMode(0);
   gStyle->SetCanvasBorderMode(0);
   gStyle->SetCanvasColor(0);

   TCanvas *c = new TCanvas("c","Graph2D example",0,0,800,800);
   c->Divide(2,3);

   Double_t rnd, x, y, z;
   Double_t e = 0.3;
   Int_t nd = 400;
   Int_t np = 10000;

   TRandom r;
   Double_t fl = 6;
   TF2  *f2 = new TF2("f2","1000*(([0]*sin(x)/x)*([1]*sin(y)/y))+200",-fl,fl,-fl,fl);
   f2->SetParameters(1,1);
   TGraph2D *dt = new TGraph2D();

   // Fill the 2D graph
   for (Int_t N=0; N<nd; N++) {
      f2->GetRandom2(x,y);
      // Generate a random number in [-e,e]
      rnd = 2*r.Rndm()*e-e;
      z = f2->Eval(x,y)*(1+rnd);
      dt->SetPoint(N,x,y,z);
   }

   Double_t hr = 350;
   TH1D *h1 = new TH1D("h1", "#splitline{Difference between Original function}{and Function with noise}", 100, -hr, hr);
   TH1D *h2 = new TH1D("h2", "#splitline{Difference between Original function}{and Interpolation with Delaunay triangles}", 100, -hr, hr);
   TH1D *h3 = new TH1D("h3", "#splitline{Difference between Original function}{and Minuit fit}", 500, -hr, hr);

   f2->SetParameters(0.5,1.5);
   dt->Fit(f2);
   TF2 *fit2 = (TF2*)dt->FindObject("f2");

   f2->SetParameters(1,1);

   for (Int_t N=0; N<np; N++) {
      f2->GetRandom2(x,y);
      // Generate a random number in [-e,e]
      rnd = 2*r.Rndm()*e-e;
      z = f2->Eval(x,y)*(1+rnd);
      h1->Fill(f2->Eval(x,y)-z);
      z = dt->Interpolate(x,y);
      h2->Fill(f2->Eval(x,y)-z);
      z = fit2->Eval(x,y);
      h3->Fill(f2->Eval(x,y)-z);
   }

   gStyle->SetPalette(1);
   c->cd(1);
   f2->SetTitle("Original function with Graph2D points on top");
   f2->Draw("surf");
   dt->Draw("p");

   c->cd(3);
   dt->SetMargin(0.1);
   dt->SetFillColor(36);
   dt->SetTitle("Histogram produced with Delaunay interpolation");
   dt->Draw("surf4");

   c->cd(5);
   fit2->SetTitle("Minuit fit result on the Graph2D points");
   fit2->Draw("surf1");

   h1->SetFillColor(47);
   h2->SetFillColor(38);
   h3->SetFillColor(29);

   c->cd(2); h1->Fit("gaus","Q") ; h1->Draw();
   c->cd(4); h2->Fit("gaus","Q") ; h2->Draw();
   c->cd(6); h3->Fit("gaus","Q") ; h3->Draw();
}
<commit_msg>Minor change to support latest version of TGraph2D<commit_after>#include <TMath.h>
#include <TGraph2D.h>
#include <TRandom.h>
#include <TStyle.h>
#include <TCanvas.h>
#include <TF2.h>

void graph2dfit()
{
   gStyle->SetOptStat(0);
   gStyle->SetOptFit();
   gStyle->SetPadBorderMode(0);
   gStyle->SetFrameBorderMode(0);
   gStyle->SetCanvasBorderMode(0);
   gStyle->SetCanvasColor(0);

   TCanvas *c = new TCanvas("c","Graph2D example",0,0,800,800);
   c->Divide(2,3);

   Double_t rnd, x, y, z;
   Double_t e = 0.3;
   Int_t nd = 400;
   Int_t np = 10000;

   TRandom r;
   Double_t fl = 6;
   TF2  *f2 = new TF2("f2","1000*(([0]*sin(x)/x)*([1]*sin(y)/y))+200",-fl,fl,-fl,fl);
   f2->SetParameters(1,1);
   TGraph2D *dt = new TGraph2D();

   // Fill the 2D graph
   for (Int_t N=0; N<nd; N++) {
      f2->GetRandom2(x,y);
      // Generate a random number in [-e,e]
      rnd = 2*r.Rndm()*e-e;
      z = f2->Eval(x,y)*(1+rnd);
      dt->SetPoint(N,x,y,z);
   }

   Double_t hr = 350;
   TH1D *h1 = new TH1D("h1", "#splitline{Difference between Original function}{and Function with noise}", 100, -hr, hr);
   TH1D *h2 = new TH1D("h2", "#splitline{Difference between Original function}{and Interpolation with Delaunay triangles}", 100, -hr, hr);
   TH1D *h3 = new TH1D("h3", "#splitline{Difference between Original function}{and Minuit fit}", 500, -hr, hr);

   f2->SetParameters(0.5,1.5);
   dt->Fit(f2);
   TF2 *fit2 = (TF2*)dt->FindObject("f2");

   f2->SetParameters(1,1);

   for (Int_t N=0; N<np; N++) {
      f2->GetRandom2(x,y);
      // Generate a random number in [-e,e]
      rnd = 2*r.Rndm()*e-e;
      z = f2->Eval(x,y)*(1+rnd);
      h1->Fill(f2->Eval(x,y)-z);
      z = dt->Interpolate(x,y);
      h2->Fill(f2->Eval(x,y)-z);
      z = fit2->Eval(x,y);
      h3->Fill(f2->Eval(x,y)-z);
   }

   gStyle->SetPalette(1);
   c->cd(1);
   f2->SetTitle("Original function with Graph2D points on top");
   f2->Draw("surf");
   dt->Draw("same p");

   c->cd(3);
   dt->SetMargin(0.1);
   dt->SetFillColor(36);
   dt->SetTitle("Histogram produced with Delaunay interpolation");
   dt->Draw("surf4");

   c->cd(5);
   fit2->SetTitle("Minuit fit result on the Graph2D points");
   fit2->Draw("surf1");

   h1->SetFillColor(47);
   h2->SetFillColor(38);
   h3->SetFillColor(29);

   c->cd(2); h1->Fit("gaus","Q") ; h1->Draw();
   c->cd(4); h2->Fit("gaus","Q") ; h2->Draw();
   c->cd(6); h3->Fit("gaus","Q") ; h3->Draw();
}
<|endoftext|>


<commit_before>// Implementation of functions declared in FLUID designer

#include "mainwindow.h"

#include <FL/Fl.H>

#include <iostream>
#include <fstream>
#include <thread>
#include <chrono>

#include "../configuration.h"
#include "../gnss_packet.h"
#include "../pichi.h"

void MainWindow::init()
{
  // Must be set or callbacks will be called with nullptr
  choice_display_device->user_data(this);

  display_add_device(Pichi::local_device_id);
  choice_display_device->value(0);
}

void MainWindow::apply_settings()
{
  Configuration conf{pichi_->config()};
  try {
    conf.device_id = std::stoul(text_device_id->value());
    conf.gnss_port = text_gnss_port->value();
    conf.gnss_port_rate = std::stoul(text_gnss_port_rate->value());
    conf.use_msg_rmc = check_rmc->value();
    conf.use_msg_gga = check_gga->value();
    conf.use_msg_gsv = check_gsv->value();
    conf.use_msg_other = check_other->value();
    conf.trans_ip = text_trans_ip->value();
    conf.trans_port = std::stoul(text_trans_port->value());
    conf.recv_ip = text_recv_ip->value();
    conf.recv_port = std::stoul(text_recv_port->value());
    pichi_->set_config(conf);
  }
  catch (const std::invalid_argument& e) {
    std::cerr << "Error applying settings: " << e.what()
              << "\nSettings were not applied." << std::endl;
  }
  catch (const std::out_of_range& e) {
    std::cerr << "Error applying settings: " << e.what()
              << "\nSettings were not applied." << std::endl;
  }
}

void MainWindow::load_settings()
{
  auto& conf = pichi_->config();
  text_device_id->value(std::to_string(conf.device_id).c_str());
  text_gnss_port->value(conf.gnss_port.c_str());
  text_gnss_port_rate->value(std::to_string(conf.gnss_port_rate).c_str());
  check_rmc->value(conf.use_msg_rmc);
  check_gga->value(conf.use_msg_gga);
  check_gsv->value(conf.use_msg_gsv);
  check_other->value(conf.use_msg_other);
  text_trans_ip->value(conf.trans_ip.c_str());
  text_trans_port->value(std::to_string(conf.trans_port).c_str());
  text_recv_ip->value(conf.recv_ip.c_str());
  text_recv_port->value(std::to_string(conf.recv_port).c_str());
}

void MainWindow::button_start_clicked()
{
  if (button_start->value()) {
    // Make sure transceiver isn't running
    if (pichi_->is_active()) {
      std::cerr << "Already running!" << std::endl;
      button_start->value(1);
    }
    else {
      switch (choice_mode->value()) {
        case 0: pichi_->start_gnss_transmitter(); break;
        case 1: pichi_->start_gnss_receiver(); break;
        case 2: pichi_->start_gnss_logger(); break;
        case 3: pichi_->start_gnss_display(); break;
        case 4: pichi_->start_debugger(); break;
      }
      button_start->label("Stop");
      last_count_ = 0;
      Fl::add_timeout(1.0, &MainWindow::update_status_callback, this);

      if (choice_mode->value() < 4) {
        Fl::add_timeout(0.5, &MainWindow::update_display_callback, this);
      }
    }
  }
  else {
    pichi_->stop();
    button_start->label("Start");
    Fl::remove_timeout(&MainWindow::update_status_callback, this);
    Fl::remove_timeout(&MainWindow::update_display_callback, this);
  }
}

void MainWindow::button_apply_clicked()
{
  apply_settings();
}

void MainWindow::button_sync_time_clicked()
{
  // This is just for convenience and not intented as a proper time sync method
  if (!pichi_->is_active()) {
    // Rough time sync via NTP
    std::cout << "Syncing time, please wait..." << std::endl;
    std::thread t{[]{ system("sudo service ntp stop && sudo ntpd -gq && sudo service ntp start"); }};
    t.detach();
  }
  else
    std::cerr << "Can't sync time while running!" << std::endl;
}

void MainWindow::radio_log_clicked()
{
  // TODO: Implement short logging
}

void MainWindow::update_status_callback(void* p)
{
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->update_status();
  Fl::repeat_timeout(1.0, &MainWindow::update_status_callback, w);
}

void MainWindow::update_status()
{
  // Rate (intented to be in hz) only works when MainWindow is updated each second
  auto count = pichi_->activity_count();
  auto rate = count - last_count_;
  last_count_ = count;

  text_rate->value(std::to_string(rate).c_str());
  text_total->value(std::to_string(count).c_str());
}

void MainWindow::update_display_callback(void* p)
{
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->update_display();
  Fl::repeat_timeout(0.5, &MainWindow::update_display_callback, w);
}

void MainWindow::update_display()
{
  bool success = false;
  gnss::LocationPacket location;
  std::tie(success, location) = pichi_->gnss_location(display_device_id_);

  if (success) {
    text_display_utc->value(std::to_string(location.utc_timestamp).c_str());
    text_display_lat->value(std::to_string(location.latitude).c_str());
    text_display_long->value(std::to_string(location.longitude).c_str());
  }

  auto ids = pichi_->new_device_ids();
  for (auto i : ids)
    display_add_device(i);
}

void MainWindow::display_device_changed_callback(Fl_Choice* o, void* p)
{
  std::cout << "rofl: " << o << " : " << p << std::endl;
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->display_device_changed(w->mapped_device_id(o->value()));
}

void MainWindow::display_device_changed(uint16_t id)
{
  display_device_id_ = id;
}

void MainWindow::display_add_device(uint16_t id)
{
  std::string name = std::to_string(id);
  choice_display_device->add(name.c_str(), 0, nullptr);
  int index = choice_display_device->find_index(name.c_str());
  mapped_device_ids_[index] = id;
}

uint16_t MainWindow::mapped_device_id(int index)
{
  return mapped_device_ids_[index];
}
<commit_msg>Removed debug output<commit_after>// Implementation of functions declared in FLUID designer

#include "mainwindow.h"

#include <FL/Fl.H>

#include <iostream>
#include <fstream>
#include <thread>
#include <chrono>

#include "../configuration.h"
#include "../gnss_packet.h"
#include "../pichi.h"

void MainWindow::init()
{
  // Must be set or callbacks will be called with nullptr
  choice_display_device->user_data(this);

  display_add_device(Pichi::local_device_id);
  choice_display_device->value(0);
}

void MainWindow::apply_settings()
{
  Configuration conf{pichi_->config()};
  try {
    conf.device_id = std::stoul(text_device_id->value());
    conf.gnss_port = text_gnss_port->value();
    conf.gnss_port_rate = std::stoul(text_gnss_port_rate->value());
    conf.use_msg_rmc = check_rmc->value();
    conf.use_msg_gga = check_gga->value();
    conf.use_msg_gsv = check_gsv->value();
    conf.use_msg_other = check_other->value();
    conf.trans_ip = text_trans_ip->value();
    conf.trans_port = std::stoul(text_trans_port->value());
    conf.recv_ip = text_recv_ip->value();
    conf.recv_port = std::stoul(text_recv_port->value());
    pichi_->set_config(conf);
  }
  catch (const std::invalid_argument& e) {
    std::cerr << "Error applying settings: " << e.what()
              << "\nSettings were not applied." << std::endl;
  }
  catch (const std::out_of_range& e) {
    std::cerr << "Error applying settings: " << e.what()
              << "\nSettings were not applied." << std::endl;
  }
}

void MainWindow::load_settings()
{
  auto& conf = pichi_->config();
  text_device_id->value(std::to_string(conf.device_id).c_str());
  text_gnss_port->value(conf.gnss_port.c_str());
  text_gnss_port_rate->value(std::to_string(conf.gnss_port_rate).c_str());
  check_rmc->value(conf.use_msg_rmc);
  check_gga->value(conf.use_msg_gga);
  check_gsv->value(conf.use_msg_gsv);
  check_other->value(conf.use_msg_other);
  text_trans_ip->value(conf.trans_ip.c_str());
  text_trans_port->value(std::to_string(conf.trans_port).c_str());
  text_recv_ip->value(conf.recv_ip.c_str());
  text_recv_port->value(std::to_string(conf.recv_port).c_str());
}

void MainWindow::button_start_clicked()
{
  if (button_start->value()) {
    // Make sure transceiver isn't running
    if (pichi_->is_active()) {
      std::cerr << "Already running!" << std::endl;
      button_start->value(1);
    }
    else {
      switch (choice_mode->value()) {
        case 0: pichi_->start_gnss_transmitter(); break;
        case 1: pichi_->start_gnss_receiver(); break;
        case 2: pichi_->start_gnss_logger(); break;
        case 3: pichi_->start_gnss_display(); break;
        case 4: pichi_->start_debugger(); break;
      }
      button_start->label("Stop");
      last_count_ = 0;
      Fl::add_timeout(1.0, &MainWindow::update_status_callback, this);

      if (choice_mode->value() < 4) {
        Fl::add_timeout(0.5, &MainWindow::update_display_callback, this);
      }
    }
  }
  else {
    pichi_->stop();
    button_start->label("Start");
    Fl::remove_timeout(&MainWindow::update_status_callback, this);
    Fl::remove_timeout(&MainWindow::update_display_callback, this);
  }
}

void MainWindow::button_apply_clicked()
{
  apply_settings();
}

void MainWindow::button_sync_time_clicked()
{
  // This is just for convenience and not intented as a proper time sync method
  if (!pichi_->is_active()) {
    // Rough time sync via NTP
    std::cout << "Syncing time, please wait..." << std::endl;
    std::thread t{[]{ system("sudo service ntp stop && sudo ntpd -gq && sudo service ntp start"); }};
    t.detach();
  }
  else
    std::cerr << "Can't sync time while running!" << std::endl;
}

void MainWindow::radio_log_clicked()
{
  // TODO: Implement short logging
}

void MainWindow::update_status_callback(void* p)
{
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->update_status();
  Fl::repeat_timeout(1.0, &MainWindow::update_status_callback, w);
}

void MainWindow::update_status()
{
  // Rate (intented to be in hz) only works when MainWindow is updated each second
  auto count = pichi_->activity_count();
  auto rate = count - last_count_;
  last_count_ = count;

  text_rate->value(std::to_string(rate).c_str());
  text_total->value(std::to_string(count).c_str());
}

void MainWindow::update_display_callback(void* p)
{
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->update_display();
  Fl::repeat_timeout(0.5, &MainWindow::update_display_callback, w);
}

void MainWindow::update_display()
{
  bool success = false;
  gnss::LocationPacket location;
  std::tie(success, location) = pichi_->gnss_location(display_device_id_);

  if (success) {
    text_display_utc->value(std::to_string(location.utc_timestamp).c_str());
    text_display_lat->value(std::to_string(location.latitude).c_str());
    text_display_long->value(std::to_string(location.longitude).c_str());
  }

  auto ids = pichi_->new_device_ids();
  for (auto i : ids)
    display_add_device(i);
}

void MainWindow::display_device_changed_callback(Fl_Choice* o, void* p)
{
  auto* w = reinterpret_cast<MainWindow*>(p);
  w->display_device_changed(w->mapped_device_id(o->value()));
}

void MainWindow::display_device_changed(uint16_t id)
{
  display_device_id_ = id;
}

void MainWindow::display_add_device(uint16_t id)
{
  std::string name = std::to_string(id);
  choice_display_device->add(name.c_str(), 0, nullptr);
  int index = choice_display_device->find_index(name.c_str());
  mapped_device_ids_[index] = id;
}

uint16_t MainWindow::mapped_device_id(int index)
{
  return mapped_device_ids_[index];
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2016 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <chrono>
#include <unordered_map>
#include <boost/intrusive/list.hpp>
#include <boost/intrusive/unordered_set.hpp>

#include <seastar/core/timer.hh>

#include "utils/exceptions.hh"

namespace bi = boost::intrusive;

namespace utils {
// Simple variant of the "LoadingCache" used for permissions in origin.

typedef lowres_clock loading_cache_clock_type;
typedef bi::list_base_hook<bi::link_mode<bi::auto_unlink>> auto_unlink_list_hook;

template<typename Tp, typename Key, typename Hash, typename EqualPred>
class timestamped_val : public auto_unlink_list_hook, public bi::unordered_set_base_hook<bi::store_hash<true>> {
public:
    typedef bi::list<timestamped_val, bi::constant_time_size<false>> lru_list_type;
    typedef Key key_type;
    typedef Tp value_type;

private:
    std::experimental::optional<Tp> _opt_value;
    loading_cache_clock_type::time_point _loaded;
    loading_cache_clock_type::time_point _last_read;
    lru_list_type& _lru_list; /// MRU item is at the front, LRU - at the back
    Key _key;

public:
    struct key_eq {
       bool operator()(const Key& k, const timestamped_val& c) const {
           return EqualPred()(k, c.key());
       }

       bool operator()(const timestamped_val& c, const Key& k) const {
           return EqualPred()(c.key(), k);
       }
    };

    timestamped_val(lru_list_type& lru_list, const Key& key)
        : _loaded(loading_cache_clock_type::now())
        , _last_read(_loaded)
        , _lru_list(lru_list)
        , _key(key) {}

    timestamped_val(lru_list_type& lru_list, Key&& key)
        : _loaded(loading_cache_clock_type::now())
        , _last_read(_loaded)
        , _lru_list(lru_list)
        , _key(std::move(key)) {}

    timestamped_val(const timestamped_val&) = default;
    timestamped_val(timestamped_val&&) = default;

    // Make sure copy/move-assignments don't go through the template below
    timestamped_val& operator=(const timestamped_val&) = default;
    timestamped_val& operator=(timestamped_val&) = default;
    timestamped_val& operator=(timestamped_val&&) = default;

    template <typename U>
    timestamped_val& operator=(U&& new_val) {
        _opt_value = std::forward<U>(new_val);
        _loaded = loading_cache_clock_type::now();
        return *this;
    }

    const Tp& value() {
        _last_read = loading_cache_clock_type::now();
        touch();
        return _opt_value.value();
    }

    explicit operator bool() const noexcept {
        return bool(_opt_value);
    }

    loading_cache_clock_type::time_point last_read() const noexcept {
        return _last_read;
    }

    loading_cache_clock_type::time_point loaded() const noexcept {
        return _loaded;
    }

    const Key& key() const {
        return _key;
    }

    friend bool operator==(const timestamped_val& a, const timestamped_val& b){
        return EqualPred()(a.key(), b.key());
    }

    friend std::size_t hash_value(const timestamped_val& v) {
        return Hash()(v.key());
    }

private:
    /// Set this item as the most recently used item.
    /// The MRU item is going to be at the front of the _lru_list, the LRU item - at the back.
    void touch() noexcept {
        auto_unlink_list_hook::unlink();
        _lru_list.push_front(*this);
    }
};

class shared_mutex {
private:
    lw_shared_ptr<semaphore> _mutex_ptr;

public:
    shared_mutex() : _mutex_ptr(make_lw_shared<semaphore>(1)) {}
    semaphore& get() const noexcept {
        return *_mutex_ptr;
    }
};

template<typename Key,
         typename Tp,
         typename Hash = std::hash<Key>,
         typename EqualPred = std::equal_to<Key>,
         typename Alloc = std::allocator<timestamped_val<Tp, Key, Hash, EqualPred>>,
         typename SharedMutexMapAlloc = std::allocator<std::pair<const Key, shared_mutex>>>
class loading_cache {
private:
    typedef timestamped_val<Tp, Key, Hash, EqualPred> ts_value_type;
    typedef bi::unordered_set<ts_value_type, bi::power_2_buckets<true>, bi::compare_hash<true>> set_type;
    typedef std::unordered_map<Key, shared_mutex, Hash, EqualPred, SharedMutexMapAlloc> write_mutex_map_type;
    typedef typename ts_value_type::lru_list_type lru_list_type;
    typedef typename set_type::bucket_traits bi_set_bucket_traits;

    static constexpr int initial_num_buckets = 256;
    static constexpr int max_num_buckets = 1024 * 1024;

public:
    typedef Tp value_type;
    typedef Key key_type;
    typedef typename set_type::iterator iterator;

    template<typename Func>
    loading_cache(size_t max_size, std::chrono::milliseconds expiry, std::chrono::milliseconds refresh, logging::logger& logger, Func&& load)
                : _buckets(initial_num_buckets)
                , _set(bi_set_bucket_traits(_buckets.data(), _buckets.size()))
                , _max_size(max_size)
                , _expiry(expiry)
                , _refresh(refresh)
                , _logger(logger)
                , _load(std::forward<Func>(load)) {

        // If expiration period is zero - caching is disabled
        if (!caching_enabled()) {
            return;
        }

        // Sanity check: if expiration period is given then non-zero refresh period and maximal size are required
        if (_refresh == std::chrono::milliseconds(0) || _max_size == 0) {
            throw exceptions::configuration_exception("loading_cache: caching is enabled but refresh period and/or max_size are zero");
        }

        _timer.set_callback([this] { on_timer(); });
        _timer.arm(_refresh);
    }

    ~loading_cache() {
        _set.clear_and_dispose([] (ts_value_type* ptr) { loading_cache::destroy_ts_value(ptr); });
    }

    future<Tp> get(const Key& k) {
        // If caching is disabled - always load in the foreground
        if (!caching_enabled()) {
            return _load(k);
        }

        // If the key is not in the cache yet, then find_or_create() is going to
        // create a new uninitialized value in the map. If the value is already
        // in the cache (the fast path) simply return the value. Otherwise, take
        // the mutex and try to load the value (the slow path).
        iterator ts_value_it = find_or_create(k);
        if (*ts_value_it) {
            return make_ready_future<Tp>(ts_value_it->value());
        } else {
            return slow_load(k);
        }
    }

private:
    bool caching_enabled() const {
        return _expiry != std::chrono::milliseconds(0);
    }

    /// Look for the entry with the given key. It it doesn't exist - create a new one and add it to the _set.
    ///
    /// \param k The key to look for
    ///
    /// \return An iterator to the value with the given key (always dirrerent from _set.end())
    template <typename KeyType>
    iterator find_or_create(KeyType&& k) {
        iterator i = _set.find(k, Hash(), typename ts_value_type::key_eq());
        if (i == _set.end()) {
            ts_value_type* new_ts_val = Alloc().allocate(1);
            new(new_ts_val) ts_value_type(_lru_list, std::forward<KeyType>(k));
            auto p = _set.insert(*new_ts_val);
            i = p.first;
        }

        return i;
    }

    static void destroy_ts_value(ts_value_type* val) {
        val->~ts_value_type();
        Alloc().deallocate(val, 1);
    }

    future<Tp> slow_load(const Key& k) {
        // If the key is not in the cache yet, then _write_mutex_map[k] is going
        // to create a new value with the initialized mutex. The mutex is going
        // to serialize the producers and only the first one is going to
        // actually issue a load operation and initialize the value with the
        // received result. The rest are going to see (and read) the initialized
        // value when they enter the critical section.
        shared_mutex sm = _write_mutex_map[k];
        return with_semaphore(sm.get(), 1, [this, k] {
            iterator ts_value_it = find_or_create(k);
            if (*ts_value_it) {
                return make_ready_future<Tp>(ts_value_it->value());
            }
            _logger.trace("{}: storing the value for the first time", k);
            return _load(k).then([this, k] (Tp t) {
                // we have to "re-read" the _set here because the value may have been evicted by now
                iterator ts_value_it = find_or_create(std::move(k));
                *ts_value_it = std::move(t);
                return make_ready_future<Tp>(ts_value_it->value());
            });
        }).finally([sm] {});
    }

    future<> reload(ts_value_type& ts_val) {
        return _load(ts_val.key()).then_wrapped([this, &ts_val] (auto&& f) {
            // The exceptions are related to the load operation itself.
            // We should ignore them for the background reads - if
            // they persist the value will age and will be reloaded in
            // the forground. If the foreground READ fails the error
            // will be propagated up to the user and will fail the
            // corresponding query.
            try {
                ts_val = f.get0();
            } catch (std::exception& e) {
                _logger.debug("{}: reload failed: {}", ts_val.key(), e.what());
            } catch (...) {
                _logger.debug("{}: reload failed: unknown error", ts_val.key());
            }
        });
    }

    void erase(iterator it) {
        _set.erase_and_dispose(it, [] (ts_value_type* ptr) { loading_cache::destroy_ts_value(ptr); });
        // no need to delete the item from _lru_list - it's auto-deleted
    }

    void drop_expired() {
        auto now = loading_cache_clock_type::now();
        _lru_list.remove_and_dispose_if([now, this] (const ts_value_type& v) {
            using namespace std::chrono;
            // An entry should be discarded if it hasn't been reloaded for too long or nobody cares about it anymore
            auto since_last_read = now - v.last_read();
            auto since_loaded = now - v.loaded();
            if (_expiry < since_last_read || _expiry < since_loaded) {
                _logger.trace("drop_expired(): {}: dropping the entry: _expiry {},  ms passed since: loaded {} last_read {}", v.key(), _expiry.count(), duration_cast<milliseconds>(since_loaded).count(), duration_cast<milliseconds>(since_last_read).count());
                return true;
            }
            return false;
        }, [this] (ts_value_type* p) {
            erase(_set.iterator_to(*p));
        });
    }

    // Shrink the cache to the _max_size discarding the least recently used items
    void shrink() {
        if (_set.size() > _max_size) {
            auto num_items_to_erase = _set.size() - _max_size;
            for (size_t i = 0; i < num_items_to_erase; ++i) {
                using namespace std::chrono;
                ts_value_type& ts_val = *_lru_list.rbegin();
                _logger.trace("shrink(): {}: dropping the entry: ms since last_read {}", ts_val.key(), duration_cast<milliseconds>(loading_cache_clock_type::now() - ts_val.last_read()).count());
                erase(_set.iterator_to(ts_val));
            }
        }
    }

    void rehash() {
        size_t new_buckets_count = 0;

        // Don't grow or shrink too fast even if there is a steep drop/growth in the number of elements in the set.
        // Exponential growth/backoff should be good enough.
        //
        // Try to keep the load factor between 0.25 and 1.0.
        if (_set.size() < _current_buckets_count / 4) {
            new_buckets_count = _current_buckets_count / 4;
        } else if (_set.size() > _current_buckets_count) {
            new_buckets_count = _current_buckets_count * 2;
        }

        if (new_buckets_count < initial_num_buckets || new_buckets_count > max_num_buckets) {
            return;
        }

        std::vector<typename set_type::bucket_type> new_buckets(new_buckets_count);
        _set.rehash(bi_set_bucket_traits(new_buckets.data(), new_buckets.size()));
        _logger.trace("rehash(): buckets count changed: {} -> {}", _current_buckets_count, new_buckets_count);

        _buckets.swap(new_buckets);
        _current_buckets_count = new_buckets_count;
    }

    void on_timer() {
        _logger.trace("on_timer(): start");

        auto timer_start_tp = loading_cache_clock_type::now();

        // Clear all cached mutexes
        _write_mutex_map.clear();

        // Clean up items that were not touched for the whole _expiry period.
        drop_expired();

        // Remove the least recently used items if map is too big.
        shrink();

        // check if rehashing is needed and do it if it is.
        rehash();

        // Reload all those which vlaue needs to be reloaded.
        parallel_for_each(_set.begin(), _set.end(), [this, curr_time = timer_start_tp] (auto& ts_val) {
            _logger.trace("on_timer(): {}: checking the value age", ts_val.key());
            if (ts_val && ts_val.loaded() + _refresh < curr_time) {
                _logger.trace("on_timer(): {}: reloading the value", ts_val.key());
                return this->reload(ts_val);
            }
            return now();
        }).finally([this, timer_start_tp] {
            _logger.trace("on_timer(): rearming");
            _timer.arm(timer_start_tp + _refresh);
        });
    }

    std::vector<typename set_type::bucket_type> _buckets;
    size_t _current_buckets_count = initial_num_buckets;
    set_type _set;
    write_mutex_map_type _write_mutex_map;
    lru_list_type _lru_list;
    size_t _max_size;
    std::chrono::milliseconds _expiry;
    std::chrono::milliseconds _refresh;
    logging::logger& _logger;
    std::function<future<Tp>(const Key&)> _load;
    timer<lowres_clock> _timer;
};

}

<commit_msg>utils::loading_cache: make a timer use a loading_cache_clock_type clock as a source<commit_after>/*
 * Copyright (C) 2016 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <chrono>
#include <unordered_map>
#include <boost/intrusive/list.hpp>
#include <boost/intrusive/unordered_set.hpp>

#include <seastar/core/timer.hh>

#include "utils/exceptions.hh"

namespace bi = boost::intrusive;

namespace utils {
// Simple variant of the "LoadingCache" used for permissions in origin.

typedef lowres_clock loading_cache_clock_type;
typedef bi::list_base_hook<bi::link_mode<bi::auto_unlink>> auto_unlink_list_hook;

template<typename Tp, typename Key, typename Hash, typename EqualPred>
class timestamped_val : public auto_unlink_list_hook, public bi::unordered_set_base_hook<bi::store_hash<true>> {
public:
    typedef bi::list<timestamped_val, bi::constant_time_size<false>> lru_list_type;
    typedef Key key_type;
    typedef Tp value_type;

private:
    std::experimental::optional<Tp> _opt_value;
    loading_cache_clock_type::time_point _loaded;
    loading_cache_clock_type::time_point _last_read;
    lru_list_type& _lru_list; /// MRU item is at the front, LRU - at the back
    Key _key;

public:
    struct key_eq {
       bool operator()(const Key& k, const timestamped_val& c) const {
           return EqualPred()(k, c.key());
       }

       bool operator()(const timestamped_val& c, const Key& k) const {
           return EqualPred()(c.key(), k);
       }
    };

    timestamped_val(lru_list_type& lru_list, const Key& key)
        : _loaded(loading_cache_clock_type::now())
        , _last_read(_loaded)
        , _lru_list(lru_list)
        , _key(key) {}

    timestamped_val(lru_list_type& lru_list, Key&& key)
        : _loaded(loading_cache_clock_type::now())
        , _last_read(_loaded)
        , _lru_list(lru_list)
        , _key(std::move(key)) {}

    timestamped_val(const timestamped_val&) = default;
    timestamped_val(timestamped_val&&) = default;

    // Make sure copy/move-assignments don't go through the template below
    timestamped_val& operator=(const timestamped_val&) = default;
    timestamped_val& operator=(timestamped_val&) = default;
    timestamped_val& operator=(timestamped_val&&) = default;

    template <typename U>
    timestamped_val& operator=(U&& new_val) {
        _opt_value = std::forward<U>(new_val);
        _loaded = loading_cache_clock_type::now();
        return *this;
    }

    const Tp& value() {
        _last_read = loading_cache_clock_type::now();
        touch();
        return _opt_value.value();
    }

    explicit operator bool() const noexcept {
        return bool(_opt_value);
    }

    loading_cache_clock_type::time_point last_read() const noexcept {
        return _last_read;
    }

    loading_cache_clock_type::time_point loaded() const noexcept {
        return _loaded;
    }

    const Key& key() const {
        return _key;
    }

    friend bool operator==(const timestamped_val& a, const timestamped_val& b){
        return EqualPred()(a.key(), b.key());
    }

    friend std::size_t hash_value(const timestamped_val& v) {
        return Hash()(v.key());
    }

private:
    /// Set this item as the most recently used item.
    /// The MRU item is going to be at the front of the _lru_list, the LRU item - at the back.
    void touch() noexcept {
        auto_unlink_list_hook::unlink();
        _lru_list.push_front(*this);
    }
};

class shared_mutex {
private:
    lw_shared_ptr<semaphore> _mutex_ptr;

public:
    shared_mutex() : _mutex_ptr(make_lw_shared<semaphore>(1)) {}
    semaphore& get() const noexcept {
        return *_mutex_ptr;
    }
};

template<typename Key,
         typename Tp,
         typename Hash = std::hash<Key>,
         typename EqualPred = std::equal_to<Key>,
         typename Alloc = std::allocator<timestamped_val<Tp, Key, Hash, EqualPred>>,
         typename SharedMutexMapAlloc = std::allocator<std::pair<const Key, shared_mutex>>>
class loading_cache {
private:
    typedef timestamped_val<Tp, Key, Hash, EqualPred> ts_value_type;
    typedef bi::unordered_set<ts_value_type, bi::power_2_buckets<true>, bi::compare_hash<true>> set_type;
    typedef std::unordered_map<Key, shared_mutex, Hash, EqualPred, SharedMutexMapAlloc> write_mutex_map_type;
    typedef typename ts_value_type::lru_list_type lru_list_type;
    typedef typename set_type::bucket_traits bi_set_bucket_traits;

    static constexpr int initial_num_buckets = 256;
    static constexpr int max_num_buckets = 1024 * 1024;

public:
    typedef Tp value_type;
    typedef Key key_type;
    typedef typename set_type::iterator iterator;

    template<typename Func>
    loading_cache(size_t max_size, std::chrono::milliseconds expiry, std::chrono::milliseconds refresh, logging::logger& logger, Func&& load)
                : _buckets(initial_num_buckets)
                , _set(bi_set_bucket_traits(_buckets.data(), _buckets.size()))
                , _max_size(max_size)
                , _expiry(expiry)
                , _refresh(refresh)
                , _logger(logger)
                , _load(std::forward<Func>(load)) {

        // If expiration period is zero - caching is disabled
        if (!caching_enabled()) {
            return;
        }

        // Sanity check: if expiration period is given then non-zero refresh period and maximal size are required
        if (_refresh == std::chrono::milliseconds(0) || _max_size == 0) {
            throw exceptions::configuration_exception("loading_cache: caching is enabled but refresh period and/or max_size are zero");
        }

        _timer.set_callback([this] { on_timer(); });
        _timer.arm(_refresh);
    }

    ~loading_cache() {
        _set.clear_and_dispose([] (ts_value_type* ptr) { loading_cache::destroy_ts_value(ptr); });
    }

    future<Tp> get(const Key& k) {
        // If caching is disabled - always load in the foreground
        if (!caching_enabled()) {
            return _load(k);
        }

        // If the key is not in the cache yet, then find_or_create() is going to
        // create a new uninitialized value in the map. If the value is already
        // in the cache (the fast path) simply return the value. Otherwise, take
        // the mutex and try to load the value (the slow path).
        iterator ts_value_it = find_or_create(k);
        if (*ts_value_it) {
            return make_ready_future<Tp>(ts_value_it->value());
        } else {
            return slow_load(k);
        }
    }

private:
    bool caching_enabled() const {
        return _expiry != std::chrono::milliseconds(0);
    }

    /// Look for the entry with the given key. It it doesn't exist - create a new one and add it to the _set.
    ///
    /// \param k The key to look for
    ///
    /// \return An iterator to the value with the given key (always dirrerent from _set.end())
    template <typename KeyType>
    iterator find_or_create(KeyType&& k) {
        iterator i = _set.find(k, Hash(), typename ts_value_type::key_eq());
        if (i == _set.end()) {
            ts_value_type* new_ts_val = Alloc().allocate(1);
            new(new_ts_val) ts_value_type(_lru_list, std::forward<KeyType>(k));
            auto p = _set.insert(*new_ts_val);
            i = p.first;
        }

        return i;
    }

    static void destroy_ts_value(ts_value_type* val) {
        val->~ts_value_type();
        Alloc().deallocate(val, 1);
    }

    future<Tp> slow_load(const Key& k) {
        // If the key is not in the cache yet, then _write_mutex_map[k] is going
        // to create a new value with the initialized mutex. The mutex is going
        // to serialize the producers and only the first one is going to
        // actually issue a load operation and initialize the value with the
        // received result. The rest are going to see (and read) the initialized
        // value when they enter the critical section.
        shared_mutex sm = _write_mutex_map[k];
        return with_semaphore(sm.get(), 1, [this, k] {
            iterator ts_value_it = find_or_create(k);
            if (*ts_value_it) {
                return make_ready_future<Tp>(ts_value_it->value());
            }
            _logger.trace("{}: storing the value for the first time", k);
            return _load(k).then([this, k] (Tp t) {
                // we have to "re-read" the _set here because the value may have been evicted by now
                iterator ts_value_it = find_or_create(std::move(k));
                *ts_value_it = std::move(t);
                return make_ready_future<Tp>(ts_value_it->value());
            });
        }).finally([sm] {});
    }

    future<> reload(ts_value_type& ts_val) {
        return _load(ts_val.key()).then_wrapped([this, &ts_val] (auto&& f) {
            // The exceptions are related to the load operation itself.
            // We should ignore them for the background reads - if
            // they persist the value will age and will be reloaded in
            // the forground. If the foreground READ fails the error
            // will be propagated up to the user and will fail the
            // corresponding query.
            try {
                ts_val = f.get0();
            } catch (std::exception& e) {
                _logger.debug("{}: reload failed: {}", ts_val.key(), e.what());
            } catch (...) {
                _logger.debug("{}: reload failed: unknown error", ts_val.key());
            }
        });
    }

    void erase(iterator it) {
        _set.erase_and_dispose(it, [] (ts_value_type* ptr) { loading_cache::destroy_ts_value(ptr); });
        // no need to delete the item from _lru_list - it's auto-deleted
    }

    void drop_expired() {
        auto now = loading_cache_clock_type::now();
        _lru_list.remove_and_dispose_if([now, this] (const ts_value_type& v) {
            using namespace std::chrono;
            // An entry should be discarded if it hasn't been reloaded for too long or nobody cares about it anymore
            auto since_last_read = now - v.last_read();
            auto since_loaded = now - v.loaded();
            if (_expiry < since_last_read || _expiry < since_loaded) {
                _logger.trace("drop_expired(): {}: dropping the entry: _expiry {},  ms passed since: loaded {} last_read {}", v.key(), _expiry.count(), duration_cast<milliseconds>(since_loaded).count(), duration_cast<milliseconds>(since_last_read).count());
                return true;
            }
            return false;
        }, [this] (ts_value_type* p) {
            erase(_set.iterator_to(*p));
        });
    }

    // Shrink the cache to the _max_size discarding the least recently used items
    void shrink() {
        if (_set.size() > _max_size) {
            auto num_items_to_erase = _set.size() - _max_size;
            for (size_t i = 0; i < num_items_to_erase; ++i) {
                using namespace std::chrono;
                ts_value_type& ts_val = *_lru_list.rbegin();
                _logger.trace("shrink(): {}: dropping the entry: ms since last_read {}", ts_val.key(), duration_cast<milliseconds>(loading_cache_clock_type::now() - ts_val.last_read()).count());
                erase(_set.iterator_to(ts_val));
            }
        }
    }

    void rehash() {
        size_t new_buckets_count = 0;

        // Don't grow or shrink too fast even if there is a steep drop/growth in the number of elements in the set.
        // Exponential growth/backoff should be good enough.
        //
        // Try to keep the load factor between 0.25 and 1.0.
        if (_set.size() < _current_buckets_count / 4) {
            new_buckets_count = _current_buckets_count / 4;
        } else if (_set.size() > _current_buckets_count) {
            new_buckets_count = _current_buckets_count * 2;
        }

        if (new_buckets_count < initial_num_buckets || new_buckets_count > max_num_buckets) {
            return;
        }

        std::vector<typename set_type::bucket_type> new_buckets(new_buckets_count);
        _set.rehash(bi_set_bucket_traits(new_buckets.data(), new_buckets.size()));
        _logger.trace("rehash(): buckets count changed: {} -> {}", _current_buckets_count, new_buckets_count);

        _buckets.swap(new_buckets);
        _current_buckets_count = new_buckets_count;
    }

    void on_timer() {
        _logger.trace("on_timer(): start");

        auto timer_start_tp = loading_cache_clock_type::now();

        // Clear all cached mutexes
        _write_mutex_map.clear();

        // Clean up items that were not touched for the whole _expiry period.
        drop_expired();

        // Remove the least recently used items if map is too big.
        shrink();

        // check if rehashing is needed and do it if it is.
        rehash();

        // Reload all those which vlaue needs to be reloaded.
        parallel_for_each(_set.begin(), _set.end(), [this, curr_time = timer_start_tp] (auto& ts_val) {
            _logger.trace("on_timer(): {}: checking the value age", ts_val.key());
            if (ts_val && ts_val.loaded() + _refresh < curr_time) {
                _logger.trace("on_timer(): {}: reloading the value", ts_val.key());
                return this->reload(ts_val);
            }
            return now();
        }).finally([this, timer_start_tp] {
            _logger.trace("on_timer(): rearming");
            _timer.arm(timer_start_tp + _refresh);
        });
    }

    std::vector<typename set_type::bucket_type> _buckets;
    size_t _current_buckets_count = initial_num_buckets;
    set_type _set;
    write_mutex_map_type _write_mutex_map;
    lru_list_type _lru_list;
    size_t _max_size;
    std::chrono::milliseconds _expiry;
    std::chrono::milliseconds _refresh;
    logging::logger& _logger;
    std::function<future<Tp>(const Key&)> _load;
    timer<loading_cache_clock_type> _timer;
};

}

<|endoftext|>


<commit_before>
/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <stdint.h>
#include <memory>
#include "bytes.hh"
#include "utils/allocation_strategy.hh"
#include <seastar/core/unaligned.hh>
#include <unordered_map>
#include <type_traits>

struct blob_storage {
    using size_type = uint32_t;
    using char_type = bytes_view::value_type;

    blob_storage** backref;
    size_type size;
    size_type frag_size;
    blob_storage* next;
    char_type data[];

    blob_storage(blob_storage** backref, size_type size, size_type frag_size) noexcept
        : backref(backref)
        , size(size)
        , frag_size(frag_size)
        , next(nullptr)
    {
        *unaligned_cast<blob_storage**>(backref) = this;
    }

    blob_storage(blob_storage&& o) noexcept
        : backref(o.backref)
        , size(o.size)
        , frag_size(o.frag_size)
        , next(o.next)
    {
        *unaligned_cast<blob_storage**>(backref) = this;
        o.next = nullptr;
        if (next) {
            next->backref = &next;
        }
        memcpy(data, o.data, frag_size);
    }
} __attribute__((packed));

// A managed version of "bytes" (can be used with LSA).
class managed_bytes {
    struct linearization_context {
        unsigned _nesting = 0;
        // Map from first blob_storage address to linearized version
        // We use the blob_storage address to be insentive to moving
        // a managed_bytes object.
        std::unordered_map<const blob_storage*, std::unique_ptr<bytes_view::value_type[]>> _state;
        void enter() {
            ++_nesting;
        }
        void leave() {
            if (!--_nesting) {
                _state.clear();
            }
        }
        void forget(const blob_storage* p) noexcept;
    };
    static thread_local linearization_context _linearization_context;
public:
    struct linearization_context_guard {
        linearization_context_guard() {
            _linearization_context.enter();
        }
        ~linearization_context_guard() {
            _linearization_context.leave();
        }
    };
private:
    static constexpr size_t max_inline_size = 15;
    struct small_blob {
        bytes_view::value_type data[max_inline_size];
        int8_t size; // -1 -> use blob_storage
    };
    union {
        blob_storage* ptr;
        small_blob small;
    } _u;
    static_assert(sizeof(small_blob) > sizeof(blob_storage*), "inline size too small");
private:
    bool external() const {
        return _u.small.size < 0;
    }
    size_t max_seg(allocation_strategy& alctr) {
        return alctr.preferred_max_contiguous_allocation() - sizeof(blob_storage);
    }
    void free_chain(blob_storage* p) noexcept {
        if (p->next && _linearization_context._nesting) {
            _linearization_context.forget(p);
        }
        auto& alctr = current_allocator();
        while (p) {
            auto n = p->next;
            alctr.destroy(p);
            p = n;
        }
    }
    const bytes_view::value_type* read_linearize() const {
        if (!external()) {
            return _u.small.data;
        } else  if (!_u.ptr->next) {
            return _u.ptr->data;
        } else {
            return do_linearize();
        }
    }
    bytes_view::value_type& value_at_index(blob_storage::size_type index) {
        if (!external()) {
            return _u.small.data[index];
        }
        blob_storage* a = _u.ptr;
        while (index >= a->frag_size) {
            index -= a->frag_size;
            a = a->next;
        }
        return a->data[index];
    }
    const bytes_view::value_type* do_linearize() const;
public:
    using size_type = blob_storage::size_type;
    struct initialized_later {};

    managed_bytes() {
        _u.small.size = 0;
    }

    managed_bytes(const blob_storage::char_type* ptr, size_type size)
        : managed_bytes(bytes_view(ptr, size)) {}

    managed_bytes(const bytes& b) : managed_bytes(static_cast<bytes_view>(b)) {}

    managed_bytes(initialized_later, size_type size) {
        if (size <= max_inline_size) {
            _u.small.size = size;
        } else {
            _u.small.size = -1;
            auto& alctr = current_allocator();
            auto maxseg = max_seg(alctr);
            auto now = std::min(size_t(size), maxseg);
            void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
                sizeof(blob_storage) + now, alignof(blob_storage));
            auto first = new (p) blob_storage(&_u.ptr, size, now);
            auto last = first;
            size -= now;
            try {
                while (size) {
                    auto now = std::min(size_t(size), maxseg);
                    void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
                        sizeof(blob_storage) + now, alignof(blob_storage));
                    last = new (p) blob_storage(&last->next, 0, now);
                    size -= now;
                }
            } catch (...) {
                free_chain(first);
                throw;
            }
        }
    }

    managed_bytes(bytes_view v) : managed_bytes(initialized_later(), v.size()) {
        if (!external()) {
            std::copy(v.begin(), v.end(), _u.small.data);
            return;
        }
        auto p = v.data();
        auto s = v.size();
        auto b = _u.ptr;
        while (s) {
            memcpy(b->data, p, b->frag_size);
            p += b->frag_size;
            s -= b->frag_size;
            b = b->next;
        }
        assert(!b);
    }

    managed_bytes(std::initializer_list<bytes::value_type> b) : managed_bytes(b.begin(), b.size()) {}

    ~managed_bytes() noexcept {
        if (external()) {
            free_chain(_u.ptr);
        }
    }

    managed_bytes(const managed_bytes& o) : managed_bytes(initialized_later(), o.size()) {
        if (!external()) {
            memcpy(data(), o.data(), size());
            return;
        }
        auto s = size();
        blob_storage* const* next_src = &o._u.ptr;
        blob_storage* blob_src = nullptr;
        size_type size_src = 0;
        size_type offs_src = 0;
        blob_storage** next_dst = &_u.ptr;
        blob_storage* blob_dst = nullptr;
        size_type size_dst = 0;
        size_type offs_dst = 0;
        while (s) {
            if (!size_src) {
                blob_src = *unaligned_cast<blob_storage**>(next_src);
                next_src = &blob_src->next;
                size_src = blob_src->frag_size;
                offs_src = 0;
            }
            if (!size_dst) {
                blob_dst = *unaligned_cast<blob_storage**>(next_dst);
                next_dst = &blob_dst->next;
                size_dst = blob_dst->frag_size;
                offs_dst = 0;
            }
            auto now = std::min(size_src, size_dst);
            memcpy(blob_dst->data + offs_dst, blob_src->data + offs_src, now);
            s -= now;
            offs_src += now; size_src -= now;
            offs_dst += now; size_dst -= now;
        }
        assert(size_src == 0 && size_dst == 0);
    }

    managed_bytes(managed_bytes&& o) noexcept
        : _u(o._u)
    {
        if (external()) {
            if (_u.ptr) {
                _u.ptr->backref = &_u.ptr;
            }
        }
        o._u.small.size = 0;
    }

    managed_bytes& operator=(managed_bytes&& o) noexcept {
        if (this != &o) {
            this->~managed_bytes();
            new (this) managed_bytes(std::move(o));
        }
        return *this;
    }

    managed_bytes& operator=(const managed_bytes& o) {
        if (this != &o) {
            managed_bytes tmp(o);
            this->~managed_bytes();
            new (this) managed_bytes(std::move(tmp));
        }
        return *this;
    }

    bool operator==(const managed_bytes& o) const {
        if (size() != o.size()) {
            return false;
        }
        if (!external()) {
            return bytes_view(*this) == bytes_view(o);
        } else {
            auto a = _u.ptr;
            auto a_data = a->data;
            auto a_remain = a->frag_size;
            a = a->next;
            auto b = o._u.ptr;
            auto b_data = b->data;
            auto b_remain = b->frag_size;
            b = b->next;
            while (a_remain || b_remain) {
                auto now = std::min(a_remain, b_remain);
                if (bytes_view(a_data, now) != bytes_view(b_data, now)) {
                    return false;
                }
                a_data += now;
                a_remain -= now;
                if (!a_remain && a) {
                    a_data = a->data;
                    a_remain = a->frag_size;
                    a = a->next;
                }
                b_data += now;
                b_remain -= now;
                if (!b_remain && b) {
                    b_data = b->data;
                    b_remain = b->frag_size;
                    b = b->next;
                }
            }
            return true;
        }
    }

    bool operator!=(const managed_bytes& o) const {
        return !(*this == o);
    }

    operator bytes_view() const {
        return { data(), size() };
    }

    bytes_view::value_type& operator[](size_type index) {
        return value_at_index(index);
    }

    const bytes_view::value_type& operator[](size_type index) const {
        return const_cast<const bytes_view::value_type&>(
                const_cast<managed_bytes*>(this)->value_at_index(index));
    }

    size_type size() const {
        if (external()) {
            return _u.ptr->size;
        } else {
            return _u.small.size;
        }
    }

    const blob_storage::char_type* begin() const {
        return data();
    }

    const blob_storage::char_type* end() const {
        return data() + size();
    }

    blob_storage::char_type* begin() {
        return data();
    }

    blob_storage::char_type* end() {
        return data() + size();
    }

    bool empty() const {
        return _u.small.size == 0;
    }

    blob_storage::char_type* data() {
        if (external()) {
            assert(!_u.ptr->next);  // must be linearized
            return _u.ptr->data;
        } else {
            return _u.small.data;
        }
    }

    const blob_storage::char_type* data() const {
        return read_linearize();
    }

    template <typename Func>
    friend std::result_of_t<Func()> with_linearized_managed_bytes(Func&& func);
};

// Run func() while ensuring that reads of managed_bytes objects are
// temporarlily linearized
template <typename Func>
inline
std::result_of_t<Func()>
with_linearized_managed_bytes(Func&& func) {
    managed_bytes::linearization_context_guard g;
    return func();
}

namespace std {

template <>
struct hash<managed_bytes> {
    size_t operator()(const managed_bytes& v) const {
        return hash<bytes_view>()(v);
    }
};

}
<commit_msg>utils/managed_bytes: add memory_usage()<commit_after>
/*
 * Copyright (C) 2015 ScyllaDB
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#pragma once

#include <stdint.h>
#include <memory>
#include "bytes.hh"
#include "utils/allocation_strategy.hh"
#include <seastar/core/unaligned.hh>
#include <unordered_map>
#include <type_traits>

struct blob_storage {
    using size_type = uint32_t;
    using char_type = bytes_view::value_type;

    blob_storage** backref;
    size_type size;
    size_type frag_size;
    blob_storage* next;
    char_type data[];

    blob_storage(blob_storage** backref, size_type size, size_type frag_size) noexcept
        : backref(backref)
        , size(size)
        , frag_size(frag_size)
        , next(nullptr)
    {
        *unaligned_cast<blob_storage**>(backref) = this;
    }

    blob_storage(blob_storage&& o) noexcept
        : backref(o.backref)
        , size(o.size)
        , frag_size(o.frag_size)
        , next(o.next)
    {
        *unaligned_cast<blob_storage**>(backref) = this;
        o.next = nullptr;
        if (next) {
            next->backref = &next;
        }
        memcpy(data, o.data, frag_size);
    }
} __attribute__((packed));

// A managed version of "bytes" (can be used with LSA).
class managed_bytes {
    struct linearization_context {
        unsigned _nesting = 0;
        // Map from first blob_storage address to linearized version
        // We use the blob_storage address to be insentive to moving
        // a managed_bytes object.
        std::unordered_map<const blob_storage*, std::unique_ptr<bytes_view::value_type[]>> _state;
        void enter() {
            ++_nesting;
        }
        void leave() {
            if (!--_nesting) {
                _state.clear();
            }
        }
        void forget(const blob_storage* p) noexcept;
    };
    static thread_local linearization_context _linearization_context;
public:
    struct linearization_context_guard {
        linearization_context_guard() {
            _linearization_context.enter();
        }
        ~linearization_context_guard() {
            _linearization_context.leave();
        }
    };
private:
    static constexpr size_t max_inline_size = 15;
    struct small_blob {
        bytes_view::value_type data[max_inline_size];
        int8_t size; // -1 -> use blob_storage
    };
    union {
        blob_storage* ptr;
        small_blob small;
    } _u;
    static_assert(sizeof(small_blob) > sizeof(blob_storage*), "inline size too small");
private:
    bool external() const {
        return _u.small.size < 0;
    }
    size_t max_seg(allocation_strategy& alctr) {
        return alctr.preferred_max_contiguous_allocation() - sizeof(blob_storage);
    }
    void free_chain(blob_storage* p) noexcept {
        if (p->next && _linearization_context._nesting) {
            _linearization_context.forget(p);
        }
        auto& alctr = current_allocator();
        while (p) {
            auto n = p->next;
            alctr.destroy(p);
            p = n;
        }
    }
    const bytes_view::value_type* read_linearize() const {
        if (!external()) {
            return _u.small.data;
        } else  if (!_u.ptr->next) {
            return _u.ptr->data;
        } else {
            return do_linearize();
        }
    }
    bytes_view::value_type& value_at_index(blob_storage::size_type index) {
        if (!external()) {
            return _u.small.data[index];
        }
        blob_storage* a = _u.ptr;
        while (index >= a->frag_size) {
            index -= a->frag_size;
            a = a->next;
        }
        return a->data[index];
    }
    const bytes_view::value_type* do_linearize() const;
public:
    using size_type = blob_storage::size_type;
    struct initialized_later {};

    managed_bytes() {
        _u.small.size = 0;
    }

    managed_bytes(const blob_storage::char_type* ptr, size_type size)
        : managed_bytes(bytes_view(ptr, size)) {}

    managed_bytes(const bytes& b) : managed_bytes(static_cast<bytes_view>(b)) {}

    managed_bytes(initialized_later, size_type size) {
        if (size <= max_inline_size) {
            _u.small.size = size;
        } else {
            _u.small.size = -1;
            auto& alctr = current_allocator();
            auto maxseg = max_seg(alctr);
            auto now = std::min(size_t(size), maxseg);
            void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
                sizeof(blob_storage) + now, alignof(blob_storage));
            auto first = new (p) blob_storage(&_u.ptr, size, now);
            auto last = first;
            size -= now;
            try {
                while (size) {
                    auto now = std::min(size_t(size), maxseg);
                    void* p = alctr.alloc(&standard_migrator<blob_storage>::object,
                        sizeof(blob_storage) + now, alignof(blob_storage));
                    last = new (p) blob_storage(&last->next, 0, now);
                    size -= now;
                }
            } catch (...) {
                free_chain(first);
                throw;
            }
        }
    }

    managed_bytes(bytes_view v) : managed_bytes(initialized_later(), v.size()) {
        if (!external()) {
            std::copy(v.begin(), v.end(), _u.small.data);
            return;
        }
        auto p = v.data();
        auto s = v.size();
        auto b = _u.ptr;
        while (s) {
            memcpy(b->data, p, b->frag_size);
            p += b->frag_size;
            s -= b->frag_size;
            b = b->next;
        }
        assert(!b);
    }

    managed_bytes(std::initializer_list<bytes::value_type> b) : managed_bytes(b.begin(), b.size()) {}

    ~managed_bytes() noexcept {
        if (external()) {
            free_chain(_u.ptr);
        }
    }

    managed_bytes(const managed_bytes& o) : managed_bytes(initialized_later(), o.size()) {
        if (!external()) {
            memcpy(data(), o.data(), size());
            return;
        }
        auto s = size();
        blob_storage* const* next_src = &o._u.ptr;
        blob_storage* blob_src = nullptr;
        size_type size_src = 0;
        size_type offs_src = 0;
        blob_storage** next_dst = &_u.ptr;
        blob_storage* blob_dst = nullptr;
        size_type size_dst = 0;
        size_type offs_dst = 0;
        while (s) {
            if (!size_src) {
                blob_src = *unaligned_cast<blob_storage**>(next_src);
                next_src = &blob_src->next;
                size_src = blob_src->frag_size;
                offs_src = 0;
            }
            if (!size_dst) {
                blob_dst = *unaligned_cast<blob_storage**>(next_dst);
                next_dst = &blob_dst->next;
                size_dst = blob_dst->frag_size;
                offs_dst = 0;
            }
            auto now = std::min(size_src, size_dst);
            memcpy(blob_dst->data + offs_dst, blob_src->data + offs_src, now);
            s -= now;
            offs_src += now; size_src -= now;
            offs_dst += now; size_dst -= now;
        }
        assert(size_src == 0 && size_dst == 0);
    }

    managed_bytes(managed_bytes&& o) noexcept
        : _u(o._u)
    {
        if (external()) {
            if (_u.ptr) {
                _u.ptr->backref = &_u.ptr;
            }
        }
        o._u.small.size = 0;
    }

    managed_bytes& operator=(managed_bytes&& o) noexcept {
        if (this != &o) {
            this->~managed_bytes();
            new (this) managed_bytes(std::move(o));
        }
        return *this;
    }

    managed_bytes& operator=(const managed_bytes& o) {
        if (this != &o) {
            managed_bytes tmp(o);
            this->~managed_bytes();
            new (this) managed_bytes(std::move(tmp));
        }
        return *this;
    }

    bool operator==(const managed_bytes& o) const {
        if (size() != o.size()) {
            return false;
        }
        if (!external()) {
            return bytes_view(*this) == bytes_view(o);
        } else {
            auto a = _u.ptr;
            auto a_data = a->data;
            auto a_remain = a->frag_size;
            a = a->next;
            auto b = o._u.ptr;
            auto b_data = b->data;
            auto b_remain = b->frag_size;
            b = b->next;
            while (a_remain || b_remain) {
                auto now = std::min(a_remain, b_remain);
                if (bytes_view(a_data, now) != bytes_view(b_data, now)) {
                    return false;
                }
                a_data += now;
                a_remain -= now;
                if (!a_remain && a) {
                    a_data = a->data;
                    a_remain = a->frag_size;
                    a = a->next;
                }
                b_data += now;
                b_remain -= now;
                if (!b_remain && b) {
                    b_data = b->data;
                    b_remain = b->frag_size;
                    b = b->next;
                }
            }
            return true;
        }
    }

    bool operator!=(const managed_bytes& o) const {
        return !(*this == o);
    }

    operator bytes_view() const {
        return { data(), size() };
    }

    bytes_view::value_type& operator[](size_type index) {
        return value_at_index(index);
    }

    const bytes_view::value_type& operator[](size_type index) const {
        return const_cast<const bytes_view::value_type&>(
                const_cast<managed_bytes*>(this)->value_at_index(index));
    }

    size_type size() const {
        if (external()) {
            return _u.ptr->size;
        } else {
            return _u.small.size;
        }
    }

    const blob_storage::char_type* begin() const {
        return data();
    }

    const blob_storage::char_type* end() const {
        return data() + size();
    }

    blob_storage::char_type* begin() {
        return data();
    }

    blob_storage::char_type* end() {
        return data() + size();
    }

    bool empty() const {
        return _u.small.size == 0;
    }

    blob_storage::char_type* data() {
        if (external()) {
            assert(!_u.ptr->next);  // must be linearized
            return _u.ptr->data;
        } else {
            return _u.small.data;
        }
    }

    const blob_storage::char_type* data() const {
        return read_linearize();
    }

    // Returns the amount of external memory used.
    size_t memory_usage() const {
        if (external()) {
            size_t mem = 0;
            blob_storage* blob = _u.ptr;
            while (blob) {
                mem += blob->frag_size + sizeof(blob_storage);
                blob = blob->next;
            }
            return mem;
        }
        return 0;
    }

    template <typename Func>
    friend std::result_of_t<Func()> with_linearized_managed_bytes(Func&& func);
};

// Run func() while ensuring that reads of managed_bytes objects are
// temporarlily linearized
template <typename Func>
inline
std::result_of_t<Func()>
with_linearized_managed_bytes(Func&& func) {
    managed_bytes::linearization_context_guard g;
    return func();
}

namespace std {

template <>
struct hash<managed_bytes> {
    size_t operator()(const managed_bytes& v) const {
        return hash<bytes_view>()(v);
    }
};

}
<|endoftext|>