<commit_before>/*************************************************************************
 *
 *  $RCSfile: outfile.cxx,v $
 *
 *  $Revision: 1.2 $
 *
 *  last change: $Author: np $ $Date: 2002-05-14 09:02:12 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#include <precomp.h>
#include "outfile.hxx"

// NOT FULLY DECLARED SERVICES
#include <cosv/file.hxx>
#include <udm/html/htmlitem.hxx>

using namespace csi;
using csi::xml::AnAttribute;

#define CRLF "\n"
    static const char s_StdStyle[] =
//          "\ta { color: #444488; }" CRLF
/*
            "\ta.syntax { font-weight:bold; color: #444488; }" CRLF
            "\ta.memberlink { font-weight:bold; color: #444488; }" CRLF
            "\ta.objchapter { font-size:16pt; font-weight:bold }" CRLF
            "\ta.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\ta.inversebig { font-size:20pt; color: #ffffff; }" CRLF
            "\th2 { font-size:18pt; }" CRLF
            "\th2.inverse { font-size:18pt; color: #ffffff; }" CRLF
            "\th3 { font-size:15pt;  }" CRLF
            "\th3.inverse { font-size:15pt; color: #ffffff; }" CRLF
            "\th4 { font-size:14pt; }" CRLF
            "\th4.inverse { font-size:14pt; color: #ffffff; }" CRLF
            "\tp { font-size:12pt; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tp.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\tp.tagtitle { font-size:14pt; font-weight:bold; margin-top:7pt; margin-bottom:3pt; }" CRLF
            "\tp.copyright {font-size: 10pt; font-style: italic; text-align: center}" CRLF
            "\tpre.inverse { color: #ffffff; }" CRLF
            "\tcode { color: #444488; }" CRLF
*/
            "\th1 { font-size:20pt; margin-top:3pt; margin-bottom:7pt; }" CRLF
            "\th2 { font-family:\"Arial\"; font-size:16pt; margin-top:3pt; margin-bottom:5pt; }" CRLF
            "\th3 { font-size:13pt; margin-top:2pt; margin-bottom:3pt; }" CRLF
            "\th4 { font-size:10pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdl { margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tdl.member { margin-top:1pt; margin-bottom:1pt; background-color:#eeeeff; }" CRLF
            "\tdt { font-size:10pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdt.member { font-size:13pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdt.simple { font-size:10pt; font-weight:normal; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdd { font-size:10pt; margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tdd.member { font-size:10pt; margin-top:1pt; margin-bottom:1pt; background-color:#ffffff; }" CRLF
            "\tp  { font-size:10pt; margin-top:3pt; margin-bottom:1pt; }" CRLF
            "\tpre { font-family: Times, serif; font-size:10pt; margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\ttr { font-size:10pt; }" CRLF
            "\ttd { font-size:10pt; }" CRLF

//          "\t.TableHeadingColor     { background: #CCCCFF } /* Dark mauve */" CRLF
//          "\t.TableSubHeadingColor  { background: #EEEEFF } /* Light mauve */" CRLF
//          "\t.TableRowColor         { background: #FFFFFF } /* White */" CRLF
//          "\t.FrameTitleFont   { font-size: normal; font-family: normal }" CRLF
//          "\t.FrameHeadingFont { font-size: normal; font-family: normal }" CRLF
//          "\t.FrameItemFont    { font-size: normal; font-family: normal }" CRLF
//          "\ttd.NavBarCell1    { background-color:#EEEEFF;}/* Light mauve */" CRLF
//          "\ttd.NavBarCell1Rev { background-color:#00008B;}/* Dark Blue */" CRLF
//          "\tfont.NavBarFont1    { font-family: Arial, Helvetica, sans-serif; color:#000000;}" CRLF
//          "\tfont.NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;}" CRLF
//          "\t.NavBarCell2    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
//          "\t.NavBarCell3    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
            ;

HtmlDocuFile::HtmlDocuFile()
//  :   // aBodyData
{
    aBodyData
//      << new AnAttribute( "link", "#0000cc" )
//      << new AnAttribute( "vlink", "#9999cc" )
        << new AnAttribute( "bgcolor", "#ffffff" );

    SetStyle(s_StdStyle);
}

void
HtmlDocuFile::SetLocation( const csv::ploc::Path & i_rFilePath )
{
    static StreamStr sPath_(1000);
    sPath_.seekp(0);
    i_rFilePath.Get( sPath_ );

    sFilePath = sPath_.c_str();
}

void
HtmlDocuFile::SetTitle( const char * i_sTitle )
{
    sTitle = i_sTitle;
}

void
HtmlDocuFile::SetStyle( const char * i_sStyle )
{
    sStyle = i_sStyle;
}

void
HtmlDocuFile::SetBodyAttr( const char *        i_sAttrName,
                            const char *        i_sAttrValue )
{
    aBodyData << new AnAttribute( i_sAttrName, i_sAttrValue );
}

void
HtmlDocuFile::SetCopyright( const char * i_sCopyright )
{
    sCopyright = i_sCopyright;
}

void
HtmlDocuFile::EmptyBody()
{
     aBodyData.SetContent(0);
    aBodyData
        >> *new html::Label( "_top_" )
        << " ";

}

bool
HtmlDocuFile::CreateFile()
{
    csv::File aFile(sFilePath, csv::CFM_CREATE);
    if (NOT aFile.open())
    {
        Cerr() << "Can't create file " << sFilePath << "." << Endl();
        return false;
    }

    WriteHeader(aFile);
    WriteBody(aFile);

    // Write end
    static const char sCompletion[] = "\n\n";
    aFile.write( sCompletion );

    aFile.close();
    Cout() << '.' << Flush();
    return true;
}

void
HtmlDocuFile::WriteHeader( csv::File & io_aFile )
{
    static const char s1[] =
        "\n\n<title>";
    static const char s2[] =
        "</title>\n\n\n";

    io_aFile.write( s1 );
    io_aFile.write( sTitle );
    io_aFile.write( s2 );
    io_aFile.write( sStyle );
    io_aFile.write( s3 );
}

void
HtmlDocuFile::WriteBody( csv::File & io_aFile )
{
    aBodyData
        >> *new html::Link( "#_top_" )
                << new html::ClassAttr( "objchapter" )
                << "Top of Page";

    if ( sCopyright.length() > 0 )
    {
        aBodyData
#ifndef COMPATIBLE_NETSCAPE_47
            >> *new html::HorizontalLine
                    <<  new html::SizeAttr( "3" );
#else
            << new xml::XmlCode("
");
#endif

        aBodyData
            >> *new html::Paragraph
                    << new html::ClassAttr( "copyright" )
                    << new xml::AnAttribute( "align", "center" )
                    << sCopyright ;
    }
    aBodyData.WriteOut(io_aFile);
}

#if 0

#define CRLF "\n"
    static const char sStyle[] =
            "\ta { color: #444488; }" CRLF
            "\ta.syntax { font-weight:bold; color: #444488; }" CRLF
            "\ta.memberlink { font-weight:bold; color: #444488; }" CRLF
            "\ta.objchapter { font-size:16pt; font-weight:bold }" CRLF
            "\ta.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\ta.inversebig { font-size:20pt; color: #ffffff; }" CRLF
            "\th2 { font-size:18pt; }" CRLF
            "\th2.inverse { font-size:18pt; color: #ffffff; }" CRLF
            "\th3 { font-size:15pt;  }" CRLF
            "\th3.inverse { font-size:15pt; color: #ffffff; }" CRLF
            "\th4 { font-size:14pt; }" CRLF
            "\th4.inverse { font-size:14pt; color: #ffffff; }" CRLF
            "\tdt { margin-top:3pt; margin-bottom:1pt; }" CRLF
            "\tdd { margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tp { font-size:12pt; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tp.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\tp.tagtitle { font-size:14pt; font-weight:bold; margin-top:7pt; margin-bottom:3pt; }" CRLF
            "\tp.copyright {font-size: 10pt; font-style: italic; text-align: center}" CRLF
            "\tpre.inverse { color: #ffffff; }" CRLF
            "\tcode { color: #444488; }" CRLF
            ".TableHeadingColor     { background: #CCCCFF } /* Dark mauve */" CRLF
            ".TableSubHeadingColor  { background: #EEEEFF } /* Light mauve */" CRLF
            ".TableRowColor         { background: #FFFFFF } /* White */" CRLF
            ".FrameTitleFont   { font-size: normal; font-family: normal }" CRLF
            ".FrameHeadingFont { font-size: normal; font-family: normal }" CRLF
            ".FrameItemFont    { font-size: normal; font-family: normal }" CRLF
            ".NavBarCell1    { background-color:#EEEEFF;}/* Light mauve */" CRLF
            ".NavBarCell1Rev { background-color:#00008B;}/* Dark Blue */" CRLF
            ".NavBarFont1    { font-family: Arial, Helvetica, sans-serif; color:#000000;}" CRLF
            ".NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;}" CRLF
            ".NavBarCell2    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
            ".NavBarCell3    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF;
    static const char sBodyAttrs[] =
            "link=\"#444488\" vlink=\"#444488\" bgcolor=\"#ffffff\"";

#endif // 0

<commit_msg>INTEGRATION: CWS adc5 (1.2.40); FILE MERGED 2003/04/09 15:23:42 np 1.2.40.1: 108787<commit_after>/*************************************************************************
 *
 *  $RCSfile: outfile.cxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: hr $ $Date: 2003-04-15 18:45:13 $
 *
 *  The Contents of this file are made available subject to the terms of
 *  either of the following licenses
 *
 *         - GNU Lesser General Public License Version 2.1
 *         - Sun Industry Standards Source License Version 1.1
 *
 *  Sun Microsystems Inc., October, 2000
 *
 *  GNU Lesser General Public License Version 2.1
 *  =============================================
 *  Copyright 2000 by Sun Microsystems, Inc.
 *  901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *
 *  Sun Industry Standards Source License Version 1.1
 *  =================================================
 *  The contents of this file are subject to the Sun Industry Standards
 *  Source License Version 1.1 (the "License"); You may not use this file
 *  except in compliance with the License. You may obtain a copy of the
 *  License at http://www.openoffice.org/license.html.
 *
 *  Software provided under this License is provided on an "AS IS" basis,
 *  WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
 *  WITHOUT LIMITATION, WARRANTIES THAT THE SOFTWARE IS FREE OF DEFECTS,
 *  MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE, OR NON-INFRINGING.
 *  See the License for the specific provisions governing your rights and
 *  obligations concerning the Software.
 *
 *  The Initial Developer of the Original Code is: Sun Microsystems, Inc.
 *
 *  Copyright: 2000 by Sun Microsystems, Inc.
 *
 *  All Rights Reserved.
 *
 *  Contributor(s): _______________________________________
 *
 *
 ************************************************************************/

#include <precomp.h>
#include "outfile.hxx"

// NOT FULLY DECLARED SERVICES
#include <cosv/file.hxx>
#include <udm/html/htmlitem.hxx>

using namespace csi;
using csi::xml::AnAttribute;

#define CRLF "\n"
    static const char s_StdStyle[] =
//          "\ta { color: #444488; }" CRLF
/*
            "\ta.syntax { font-weight:bold; color: #444488; }" CRLF
            "\ta.memberlink { font-weight:bold; color: #444488; }" CRLF
            "\ta.objchapter { font-size:16pt; font-weight:bold }" CRLF
            "\ta.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\ta.inversebig { font-size:20pt; color: #ffffff; }" CRLF
            "\th2 { font-size:18pt; }" CRLF
            "\th2.inverse { font-size:18pt; color: #ffffff; }" CRLF
            "\th3 { font-size:15pt;  }" CRLF
            "\th3.inverse { font-size:15pt; color: #ffffff; }" CRLF
            "\th4 { font-size:14pt; }" CRLF
            "\th4.inverse { font-size:14pt; color: #ffffff; }" CRLF
            "\tp { font-size:12pt; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tp.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\tp.tagtitle { font-size:14pt; font-weight:bold; margin-top:7pt; margin-bottom:3pt; }" CRLF
            "\tp.copyright {font-size: 10pt; font-style: italic; text-align: center}" CRLF
            "\tpre.inverse { color: #ffffff; }" CRLF
            "\tcode { color: #444488; }" CRLF
*/
            "\th1 { font-size:20pt; margin-top:3pt; margin-bottom:7pt; }" CRLF
            "\th2 { font-family:\"Arial\"; font-size:16pt; margin-top:3pt; margin-bottom:5pt; }" CRLF
            "\th3 { font-size:13pt; margin-top:2pt; margin-bottom:3pt; }" CRLF
            "\th4 { font-size:10pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdl { margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tdl.member { margin-top:1pt; margin-bottom:1pt; background-color:#eeeeff; }" CRLF
            "\tdt { font-size:10pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdt.member { font-size:13pt; font-weight:bold; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdt.simple { font-size:10pt; font-weight:normal; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tdd { font-size:10pt; margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tdd.member { font-size:10pt; margin-top:1pt; margin-bottom:1pt; background-color:#ffffff; }" CRLF
            "\tp  { font-size:10pt; margin-top:3pt; margin-bottom:1pt; }" CRLF
            "\tpre { font-family: Times, serif; font-size:10pt; margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\ttr { font-size:10pt; }" CRLF
            "\ttd { font-size:10pt; }" CRLF

//          "\t.TableHeadingColor     { background: #CCCCFF } /* Dark mauve */" CRLF
//          "\t.TableSubHeadingColor  { background: #EEEEFF } /* Light mauve */" CRLF
//          "\t.TableRowColor         { background: #FFFFFF } /* White */" CRLF
//          "\t.FrameTitleFont   { font-size: normal; font-family: normal }" CRLF
//          "\t.FrameHeadingFont { font-size: normal; font-family: normal }" CRLF
//          "\t.FrameItemFont    { font-size: normal; font-family: normal }" CRLF
//          "\ttd.NavBarCell1    { background-color:#EEEEFF;}/* Light mauve */" CRLF
//          "\ttd.NavBarCell1Rev { background-color:#00008B;}/* Dark Blue */" CRLF
//          "\tfont.NavBarFont1    { font-family: Arial, Helvetica, sans-serif; color:#000000;}" CRLF
//          "\tfont.NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;}" CRLF
//          "\t.NavBarCell2    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
//          "\t.NavBarCell3    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
            ;

HtmlDocuFile::HtmlDocuFile()
//  :   // aBodyData
{
    aBodyData
//      << new AnAttribute( "link", "#0000cc" )
//      << new AnAttribute( "vlink", "#9999cc" )
        << new AnAttribute( "bgcolor", "#ffffff" );

    SetStyle(s_StdStyle);
}

void
HtmlDocuFile::SetLocation( const csv::ploc::Path & i_rFilePath )
{
    static StreamStr sPath_(1000);
    sPath_.seekp(0);
    i_rFilePath.Get( sPath_ );

    sFilePath = sPath_.c_str();
}

void
HtmlDocuFile::SetTitle( const char * i_sTitle )
{
    sTitle = i_sTitle;
}

void
HtmlDocuFile::SetStyle( const char * i_sStyle )
{
    sStyle = i_sStyle;
}

void
HtmlDocuFile::SetBodyAttr( const char *        i_sAttrName,
                            const char *        i_sAttrValue )
{
    aBodyData << new AnAttribute( i_sAttrName, i_sAttrValue );
}

void
HtmlDocuFile::SetCopyright( const char * i_sCopyright )
{
    sCopyright = i_sCopyright;
}

void
HtmlDocuFile::EmptyBody()
{
     aBodyData.SetContent(0);
    aBodyData
        >> *new html::Label( "_top_" )
        << " ";

}

bool
HtmlDocuFile::CreateFile()
{
    csv::File aFile(sFilePath, csv::CFM_CREATE);
    if (NOT aFile.open())
    {
        Cerr() << "Can't create file " << sFilePath << "." << Endl();
        return false;
    }

    WriteHeader(aFile);
    WriteBody(aFile);

    // Write end
    static const char sCompletion[] = "\n\n";
    aFile.write( sCompletion );

    aFile.close();
    Cout() << '.' << Flush();
    return true;
}

void
HtmlDocuFile::WriteHeader( csv::File & io_aFile )
{
    static const char s1[] =
        "\n\n<title>";
    static const char s2[] =
        "</title>\n\n\n";

    io_aFile.write( s1 );
    io_aFile.write( sTitle );
    io_aFile.write( s2 );
    io_aFile.write( sStyle );
    io_aFile.write( s3 );
}

void
HtmlDocuFile::WriteBody( csv::File & io_aFile )
{
    aBodyData
        >> *new html::Link( "#_top_" )
                << new html::ClassAttr( "objchapter" )
                << "Top of Page";

    if ( sCopyright.length() > 0 )
    {
        aBodyData
#ifndef COMPATIBLE_NETSCAPE_47
            >> *new html::HorizontalLine
                    <<  new html::SizeAttr( "3" );
#else
            << new xml::XmlCode("
");
#endif

        aBodyData
            >> *new html::Paragraph
                    << new html::ClassAttr( "copyright" )
                    << new xml::AnAttribute( "align", "center" )
                    << new xml::XmlCode(sCopyright);
    }
    aBodyData.WriteOut(io_aFile);
}

#if 0

#define CRLF "\n"
    static const char sStyle[] =
            "\ta { color: #444488; }" CRLF
            "\ta.syntax { font-weight:bold; color: #444488; }" CRLF
            "\ta.memberlink { font-weight:bold; color: #444488; }" CRLF
            "\ta.objchapter { font-size:16pt; font-weight:bold }" CRLF
            "\ta.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\ta.inversebig { font-size:20pt; color: #ffffff; }" CRLF
            "\th2 { font-size:18pt; }" CRLF
            "\th2.inverse { font-size:18pt; color: #ffffff; }" CRLF
            "\th3 { font-size:15pt;  }" CRLF
            "\th3.inverse { font-size:15pt; color: #ffffff; }" CRLF
            "\th4 { font-size:14pt; }" CRLF
            "\th4.inverse { font-size:14pt; color: #ffffff; }" CRLF
            "\tdt { margin-top:3pt; margin-bottom:1pt; }" CRLF
            "\tdd { margin-top:1pt; margin-bottom:1pt; }" CRLF
            "\tp { font-size:12pt; margin-top:2pt; margin-bottom:1pt; }" CRLF
            "\tp.inverse { font-size:12pt; color: #ffffff; }" CRLF
            "\tp.tagtitle { font-size:14pt; font-weight:bold; margin-top:7pt; margin-bottom:3pt; }" CRLF
            "\tp.copyright {font-size: 10pt; font-style: italic; text-align: center}" CRLF
            "\tpre.inverse { color: #ffffff; }" CRLF
            "\tcode { color: #444488; }" CRLF
            ".TableHeadingColor     { background: #CCCCFF } /* Dark mauve */" CRLF
            ".TableSubHeadingColor  { background: #EEEEFF } /* Light mauve */" CRLF
            ".TableRowColor         { background: #FFFFFF } /* White */" CRLF
            ".FrameTitleFont   { font-size: normal; font-family: normal }" CRLF
            ".FrameHeadingFont { font-size: normal; font-family: normal }" CRLF
            ".FrameItemFont    { font-size: normal; font-family: normal }" CRLF
            ".NavBarCell1    { background-color:#EEEEFF;}/* Light mauve */" CRLF
            ".NavBarCell1Rev { background-color:#00008B;}/* Dark Blue */" CRLF
            ".NavBarFont1    { font-family: Arial, Helvetica, sans-serif; color:#000000;}" CRLF
            ".NavBarFont1Rev { font-family: Arial, Helvetica, sans-serif; color:#FFFFFF;}" CRLF
            ".NavBarCell2    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF
            ".NavBarCell3    { font-family: Arial, Helvetica, sans-serif; background-color:#FFFFFF;}" CRLF;
    static const char sBodyAttrs[] =
            "link=\"#444488\" vlink=\"#444488\" bgcolor=\"#ffffff\"";

#endif // 0

<|endoftext|>


<commit_before>#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/Debug.h"
using namespace llvm;
namespace {
class BoUpSLP {
public:
  typedef SmallVector<Value *, 8> ValueList;
  BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
          TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
          DominatorTree *Dt, AssumptionCache *AC)
      : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
        Builder(Se->getContext()) {
  }
  void getTreeCost();
  void buildTree(ArrayRef<Value *> Roots,
                 ArrayRef<Value *> UserIgnoreLst = None);
  bool shouldReorder() const {
  }
  struct TreeEntry {
    ValueList Scalars;
  };
  std::vector<TreeEntry> VectorizableTree;
  SmallDenseMap<Value*, int> ScalarToTreeEntry;
  struct ExternalUser {
    Value *Scalar;
    int Lane;
  };
  typedef SmallVector<ExternalUser, 16> UserList;
  UserList ExternalUses;
  SetVector<Instruction *> GatherSeq;
  struct ScheduleData {
  };
  struct BlockScheduling {
    ScheduleData *getScheduleData(Value *V) {
    }
    template <typename ReadyListType>
    void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
    }
  };
  int NumLoadsWantToKeepOrder;
  int NumLoadsWantToChangeOrder;
  Function *F;
  TargetTransformInfo *TTI;
  TargetLibraryInfo *TLI;
  DominatorTree *DT;
  IRBuilder<> Builder;
};
void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
                        ArrayRef<Value *> UserIgnoreLst) {
  for (int EIdx = 0, EE = VectorizableTree.size(); EIdx < EE; ++EIdx) {
    TreeEntry *Entry = &VectorizableTree[EIdx];
    for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
      Value *Scalar = Entry->Scalars[Lane];
      for (User *U : Scalar->users()) {
        if (ScalarToTreeEntry.count(U)) {
        }
      }
    }
  }
}
void BoUpSLP::getTreeCost() {
  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
  int ExtractCost = 0;
  for (UserList::iterator I = ExternalUses.begin(), E = ExternalUses.end();
       I != E; ++I) {
    VectorType *VecTy = VectorType::get(I->Scalar->getType(), BundleWidth);
    ExtractCost += TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy,
                                           I->Lane);
  }
}
struct SLPVectorizer : public FunctionPass {
  static char ID;
  explicit SLPVectorizer() : FunctionPass(ID) {
  }
  ScalarEvolution *SE;
  TargetTransformInfo *TTI;
  TargetLibraryInfo *TLI;
  AliasAnalysis *AA;
  LoopInfo *LI;
  DominatorTree *DT;
  AssumptionCache *AC;
  bool runOnFunction(Function &F) override {
    SE = &getAnalysis<ScalarEvolution>();
    TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
    bool Changed = false;
    if (!TTI->getNumberOfRegisters(true))
      return false;
    BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC);
    for (auto BB : post_order(&F.getEntryBlock())) {
      Changed |= vectorizeChainsInBlock(BB, R);
    }
  }
  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<ScalarEvolution>();
    AU.addRequired<TargetTransformInfoWrapperPass>();
  }
  bool tryToVectorizePair(Value *A, Value *B, BoUpSLP &R);
  bool tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
                          ArrayRef<Value *> BuildVector = None,
                          bool allowReorder = false);
  bool tryToVectorize(BinaryOperator *V, BoUpSLP &R);
  bool vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R);
};
bool SLPVectorizer::tryToVectorizePair(Value *A, Value *B, BoUpSLP &R) {
  Value *VL[] = { A, B };
  return tryToVectorizeList(VL, R, None, true);
}
bool SLPVectorizer::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
                                       ArrayRef<Value *> BuildVector,
                                       bool allowReorder) {
  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
  if (!I0)
    return false;
  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
    unsigned OpsWidth = 0;
    DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
                 << "\n");
    ArrayRef<Value *> Ops = VL.slice(i, OpsWidth);
    if (allowReorder && R.shouldReorder()) {
      Value *ReorderedOps[] = { Ops[1], Ops[0] };
      R.buildTree(ReorderedOps, None);
    }
    R.getTreeCost();
  }
}
bool SLPVectorizer::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
  BinaryOperator *A = dyn_cast<BinaryOperator>(V->getOperand(0));
  BinaryOperator *B = dyn_cast<BinaryOperator>(V->getOperand(1));
  if (B && B->hasOneUse()) {
    BinaryOperator *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
    if (tryToVectorizePair(A0, B, R)) {
    }
  }
}
class HorizontalReduction {
public:
  bool matchAssociativeReduction(PHINode *Phi, BinaryOperator *B) {
    SmallVector<std::pair<BinaryOperator *, unsigned>, 32> Stack;
    while (!Stack.empty()) {
    }
  }
  bool tryToReduce(BoUpSLP &V, TargetTransformInfo *TTI) {
  }
};
bool SLPVectorizer::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
  bool HaveVectorizedPhiNodes = true;
  while (HaveVectorizedPhiNodes) {
    HaveVectorizedPhiNodes = false;
    for (BasicBlock::iterator instr = BB->begin(), ie = BB->end(); instr != ie;
         ++instr) {
    }
  }
  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
    if (PHINode *P = dyn_cast<PHINode>(it)) {
      Value *Rdx =
          (P->getIncomingBlock(0) == BB
               ? (P->getIncomingValue(0))
               : (P->getIncomingBlock(1) == BB ? P->getIncomingValue(1)
                                               : nullptr));
      BinaryOperator *BI = dyn_cast_or_null<BinaryOperator>(Rdx);
     Value *Inst = BI->getOperand(0);
      if (tryToVectorize(dyn_cast<BinaryOperator>(Inst), R)) {
      }
    }
      if (StoreInst *SI = dyn_cast<StoreInst>(it))
        if (BinaryOperator *BinOp =
                dyn_cast<BinaryOperator>(SI->getValueOperand())) {
          HorizontalReduction HorRdx;
          if (((HorRdx.matchAssociativeReduction(nullptr, BinOp) &&
                HorRdx.tryToReduce(R, TTI)) ||
               tryToVectorize(BinOp, R))) {
          }
        }
    if (dyn_cast<CmpInst>(it)) {
      SmallVector<Value *, 16> BuildVector;
      SmallVector<Value *, 16> BuildVectorOpds;
      tryToVectorizeList(BuildVectorOpds, R, BuildVector);
    }
  }
}
} // end anonymous namespace
char SLPVectorizer::ID = 0;
static const char lv_name[] = "SLP Vectorizer";
INITIALIZE_PASS_BEGIN(SLPVectorizer, "slp-vectorizer", lv_name, false, false)
INITIALIZE_PASS_END(SLPVectorizer, "slp-vectorizer", lv_name, false, false)
namespace llvm {
Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }
}
<commit_msg>more manual reductions<commit_after>#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
using namespace llvm;
namespace {
class BoUpSLP {
public:
  typedef SmallVector<Value *, 8> ValueList;
  BoUpSLP(Function *Func, ScalarEvolution *Se, TargetTransformInfo *Tti,
          TargetLibraryInfo *TLi, AliasAnalysis *Aa, LoopInfo *Li,
          DominatorTree *Dt, AssumptionCache *AC)
      : NumLoadsWantToKeepOrder(0), NumLoadsWantToChangeOrder(0), F(Func),
        Builder(Se->getContext()) {}
  void getTreeCost();
  void buildTree(ArrayRef<Value *> Roots,
                 ArrayRef<Value *> UserIgnoreLst = None);
  bool shouldReorder() const {}
  struct TreeEntry {
    ValueList Scalars;
  };
  std::vector<TreeEntry> VectorizableTree;
  SmallDenseMap<Value *, int> ScalarToTreeEntry;
  struct ExternalUser {
    Value *Scalar;
    int Lane;
  };
  typedef SmallVector<ExternalUser, 16> UserList;
  UserList ExternalUses;
  SetVector<Instruction *> GatherSeq;
  int NumLoadsWantToKeepOrder;
  int NumLoadsWantToChangeOrder;
  Function *F;
  TargetTransformInfo *TTI;
  TargetLibraryInfo *TLI;
  DominatorTree *DT;
  IRBuilder<> Builder;
};
void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
                        ArrayRef<Value *> UserIgnoreLst) {
  for (int EIdx = 0, EE = VectorizableTree.size(); EIdx < EE; ++EIdx) {
    TreeEntry *Entry = &VectorizableTree[EIdx];
    for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
      for (User *U : Entry->Scalars[Lane]->users()) {
      }
    }
  }
}
void BoUpSLP::getTreeCost() {
  unsigned BundleWidth = VectorizableTree[0].Scalars.size();
  for (UserList::iterator I = ExternalUses.begin(), E = ExternalUses.end();
       I != E; ++I) {
    VectorType::get(I->Scalar->getType(), BundleWidth);
  }
}
struct SLPVectorizer : public FunctionPass {
  static char ID;
  explicit SLPVectorizer() : FunctionPass(ID) {}
  ScalarEvolution *SE;
  TargetTransformInfo *TTI;
  TargetLibraryInfo *TLI;
  AliasAnalysis *AA;
  LoopInfo *LI;
  DominatorTree *DT;
  AssumptionCache *AC;
  bool runOnFunction(Function &F) override {
    SE = &getAnalysis<ScalarEvolution>();
    TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
    if (!TTI->getNumberOfRegisters(true))
      return false;
    BoUpSLP R(&F, SE, TTI, TLI, AA, LI, DT, AC);
    for (auto BB : post_order(&F.getEntryBlock())) {
      vectorizeChainsInBlock(BB, R);
    }
  }
  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<ScalarEvolution>();
    AU.addRequired<TargetTransformInfoWrapperPass>();
  }
  void tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
                          ArrayRef<Value *> BuildVector = None,
                          bool allowReorder = false);
  void tryToVectorize(BinaryOperator *V, BoUpSLP &R);
  void vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R);
};
void SLPVectorizer::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
                                       ArrayRef<Value *> BuildVector,
                                       bool allowReorder) {
  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
  if (!I0)
    return;
  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
    unsigned OpsWidth = 0;
    ArrayRef<Value *> Ops = VL.slice(i, OpsWidth);
    if (allowReorder && R.shouldReorder()) {
      Value *ReorderedOps[] = {Ops[1], Ops[0]};
      R.buildTree(ReorderedOps, None);
    }
    R.getTreeCost();
  }
}
void SLPVectorizer::tryToVectorize(BinaryOperator *V, BoUpSLP &R) {
  BinaryOperator *A = dyn_cast<BinaryOperator>(V->getOperand(0));
  BinaryOperator *B = dyn_cast<BinaryOperator>(V->getOperand(1));
  if (B && B->hasOneUse()) {
    BinaryOperator *A0 = dyn_cast<BinaryOperator>(A->getOperand(0));
    tryToVectorizeList({A0, B}, R, None, true);
  }
}
void SLPVectorizer::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; it++) {
    if (PHINode *P = dyn_cast<PHINode>(it)) {
      Value *Rdx = (P->getIncomingBlock(0) == BB
                        ? (P->getIncomingValue(0))
                        : (P->getIncomingBlock(1) == BB ? P->getIncomingValue(1)
                                                        : nullptr));
      BinaryOperator *BI = dyn_cast_or_null<BinaryOperator>(Rdx);
      tryToVectorize(dyn_cast<BinaryOperator>(BI->getOperand(0)), R);
    }
    if (StoreInst *SI = dyn_cast<StoreInst>(it))
      if (BinaryOperator *BinOp =
              dyn_cast<BinaryOperator>(SI->getValueOperand())) {
        tryToVectorize(BinOp, R);
      }
    if (dyn_cast<CmpInst>(it)) {
      SmallVector<Value *, 16> BuildVector;
      SmallVector<Value *, 16> BuildVectorOpds;
      tryToVectorizeList(BuildVectorOpds, R, BuildVector);
    }
  }
}
} // end anonymous namespace
char SLPVectorizer::ID = 0;
static const char lv_name[] = "SLP Vectorizer";
INITIALIZE_PASS_BEGIN(SLPVectorizer, "slp-vectorizer", lv_name, false, false)
INITIALIZE_PASS_END(SLPVectorizer, "slp-vectorizer", lv_name, false, false)
namespace llvm {
Pass *createSLPVectorizerPass() { return new SLPVectorizer(); }
}
<|endoftext|>


<commit_before>/*
 * ImageHasher - C++ version of the Hashing module from SimilarImage
 * Copyright (C) 2014  Nicholas Wright
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Database.cpp
 *
 *  Created on: 21 Aug 2013
 *      Author: nicholas
 */

#include "../include/Database.hpp"
#include <string>

#include <odb/transaction.hxx>
#include <odb/result.hxx>
#include <odb/schema-catalog.hxx>
#include <odb/sqlite/exceptions.hxx>
#include <odb/exceptions.hxx>

#include "table/ImageRecord.hpp"
#include "table/ImageRecord-odb.hxx"

#include "table/FilterRecord.hpp"
#include "table/FilterRecord-odb.hxx"

const char *dbName = "imageHasher.db";

const char *prune_hash_table_query = "DELETE FROM hash WHERE hash_id IN (SELECT hash_id FROM (SELECT imagerecord.hash, hash.hash_id FROM hash LEFT OUTER JOIN imagerecord  ON hash = hash_id) WHERE hash IS null);";

using namespace odb;
using namespace imageHasher::db::table;

Database::Database(const char* dbPath) {
	dbName = dbPath;
	init();
}

Database::Database() {
	init();
}

Database::~Database() {
	shutdown();
	delete this->prep_query;
	delete orm_db;
}

int Database::flush() {
	int drainCount = 0;

	LOG4CPLUS_INFO(logger, "Flushing lists...");

	drainCount = drain();
	drainCount += drain();

	return drainCount;
}

void Database::shutdown() {
	if (running) {
		LOG4CPLUS_INFO(logger, "Shutting down...");
		running = false;
		LOG4CPLUS_INFO(logger, "Waiting for db worker to finish...");
		workerThread->interrupt();
		workerThread->join();
		delete(workerThread);
		LOG4CPLUS_INFO(logger, "Closing database...");
	}
}

void Database::exec(const char* command) {
	orm_db->execute(command);
}

void Database::init() {
	boost::mutex::scoped_lock lock(dbMutex);
	this->currentList = &dataA;
	this->recordsWritten = 0;
	this->running = true;
	logger = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("Database"));

	setupDatabase();
	prepareStatements();

	workerThread = new boost::thread(&Database::doWork, this);
}

bool Database::is_db_initialised() {
	Hash hash;
	bool exists = true;
	transaction t(orm_db->begin());
	try {
		exists = orm_db->find(1, hash);
	} catch (odb::sqlite::database_exception& e) {
		exists = false;
	}
	t.commit();
	return exists;
}

void Database::initialise_db() {
	transaction t(orm_db->begin());
	odb::schema_catalog::create_schema(*orm_db, "", false);
	t.commit();
	addHashEntry("", 0);
}

void Database::setupDatabase() {
	orm_db = new sqlite::database(dbName,SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);

	LOG4CPLUS_INFO(logger, "Setting up database " << dbName);

	bool init_done = is_db_initialised();

	if(! init_done) {
		initialise_db();
	}

	transaction t_pragma (orm_db->begin());
		orm_db->execute("PRAGMA page_size = 4096;");
		orm_db->execute("PRAGMA cache_size=10000;");
		orm_db->execute("PRAGMA locking_mode=EXCLUSIVE;");
		orm_db->execute("PRAGMA temp_store = MEMORY;");
		orm_db->execute("PRAGMA journal_mode=MEMORY;");
	t_pragma.commit();

	// perform out of transaction execution
	connection_ptr c (orm_db->connection());
	c->execute("PRAGMA synchronous=NORMAL;");
}

void Database::add(db_data data) {
	boost::mutex::scoped_lock lock(flipMutex);
	currentList->push_back(data);
}

void Database::flipLists() {
	boost::mutex::scoped_lock lock(flipMutex);

	if(currentList == &dataA) {
		currentList = &dataB;
	}else{
		currentList = &dataA;
	}
}

int Database::drain() {
	boost::mutex::scoped_lock lock(dbMutex);
	std::list<db_data>* workList;
	int drainCount = 0;

	workList = currentList;
	flipLists();

	for(std::list<db_data>::iterator ite = workList->begin(); ite != workList->end(); ++ite) {
		addToBatch(*ite);
		recordsWritten++;
		drainCount++;
	}

	workList->clear();
	return drainCount;
}

bool Database::entryExists(fs::path filePath) {
	LOG4CPLUS_DEBUG(logger, "Looking for file " << filePath);

	ImageRecord ir = get_imagerecord(filePath);

	return ir.is_valid();
}

bool Database::entryExists(db_data data) {
	return entryExists(data.filePath);
}

std::list<fs::path> Database::getFilesWithPath(fs::path directoryPath) {
	std::list<fs::path> filePaths;
	std::string path_query(directoryPath.string());
	path_query += "%";

	LOG4CPLUS_INFO(logger, "Looking for files with path " << directoryPath);

	transaction t (orm_db->begin());
	std::string *param;

	odb::prepared_query<ImageRecord> pq = prep_query->get_files_with_path_query(param);

	*param = path_query;

	odb::result<ImageRecord> r(pq.execute());

	for(result<ImageRecord>::iterator itr (r.begin()); itr != r.end(); ++itr) {
		filePaths.push_back(fs::path(itr->getPath()));
	}

	t.commit();

	LOG4CPLUS_INFO(logger, "Found  " << filePaths.size() << " records for path " << directoryPath);
	return filePaths;
}

void Database::prunePath(std::list<fs::path> filePaths) {
	LOG4CPLUS_INFO(logger, "Pruning " << filePaths.size() << " path(s) from the database.");

	for(std::list<fs::path>::iterator ite = filePaths.begin(); ite != filePaths.end(); ++ite){
		ImageRecord ir = get_imagerecord(*ite);

		transaction t (orm_db->begin());

		if(ir.is_valid()) {
			orm_db->erase(ir);
		}

		t.commit();
	}
}

void Database::addToBatch(db_data data) {
	switch (data.status) {
	case OK:
		add_record(data);
		break;

	case INVALID:
		add_invalid(data);
		break;

	case FILTER:
		add_filter(data);
		break;

	default:
		LOG4CPLUS_ERROR(logger, "Unhandled state encountered");
		throw "Unhandled state encountered";
		break;
	}
}

void Database::add_record(db_data data) {
	Hash hash = get_hash(data.sha256);

	if (!hash.is_valid()) {
		hash = addHashEntry(data.sha256, data.pHash);
	}

	if(entryExists(data)) {
		LOG4CPLUS_DEBUG(logger, "Entry for " << data.filePath << " already exists, discarding...");
		recordsWritten--;
		return;
	}

	try{
	transaction t(orm_db->begin());

	ImageRecord ir = ImageRecord(data.filePath.string(), &hash);
	orm_db->persist(ir);

	t.commit();
	} catch (const odb::exception &e) {
		LOG4CPLUS_ERROR(logger, "Failed to add ImageRecord for path " << data.filePath << " : " << e.what());
	}
}

int Database::add_path_placeholder(std::string path) {

	transaction t (orm_db->begin());

	ImageRecord ir (path,NULL);
	int id = orm_db->persist(ir);

	t.commit();

	return id;
}

void Database::add_invalid(db_data data) {
	LOG4CPLUS_DEBUG(logger, "File with path " << data.filePath << " is invalid");
	recordsWritten--;
	Hash hash = get_hash("");
	transaction t(orm_db->begin());
	ImageRecord ir = ImageRecord(data.filePath.string(), &hash);
	orm_db->persist(ir);

	t.commit();
}

void Database::add_filter(db_data data) {
	LOG4CPLUS_DEBUG(logger, "Adding filter record for pHash" << data.pHash << " with reason " << data.reason);

	imageHasher::db::table::FilterRecord fr(data.pHash, data.reason);

	transaction t (orm_db->begin());

	orm_db->persist(fr);

	t.commit();
}

void Database::prepareStatements() {
	LOG4CPLUS_INFO(logger, "Creating prepared statements...");
	this->prep_query = new imageHasher::db::PreparedQuery(this->orm_db);
}

void Database::doWork() {
	while(running) {
		try{
			boost::this_thread::sleep_for(boost::chrono::seconds(3));
		}catch(boost::thread_interrupted&) {
			LOG4CPLUS_INFO(logger,"DB thread interrupted");
		}

		if(currentList->size() > 1000) {
			int drainCount = drain();
			LOG4CPLUS_INFO(logger, drainCount << " records processed, Total: " << recordsWritten);
		}
	}

	// make sure both lists are committed
	int drainCount = flush();
	LOG4CPLUS_INFO(logger, drainCount << " records processed, Total: " << recordsWritten);
}

unsigned int Database::getRecordsWritten() {
	return recordsWritten;
}

std::string Database::getSHA(fs::path filepath) {
	std::string sha = "";
	LOG4CPLUS_DEBUG(logger, "Getting SHA for path " << filepath);

	ImageRecord ir = get_imagerecord(filepath);

	if(ir.is_valid()) {
		Hash hash = ir.get_hash();
		sha = hash.get_sha256();
	}

	return sha;
}

bool Database::sha_exists(std::string sha) {
	Hash hash = get_hash(sha);
	return hash.is_valid();
}

int64_t Database::getPhash(fs::path filepath) {
	LOG4CPLUS_DEBUG(logger, "Getting pHash for path " << filepath);

	ImageRecord ir = get_imagerecord(filepath);

	if(ir.is_valid()) {
		return ir.get_hash().get_pHash();
	}

	return -1;
}

imageHasher::db::table::Hash Database::get_hash(std::string sha) {
	Hash hash;
	LOG4CPLUS_DEBUG(logger, "Getting hash for sha " << sha);

	transaction t (orm_db->begin());

	std::string *param;

	odb::prepared_query<Hash> pq = prep_query->get_hash_query(param);

	*param = sha;

	result<Hash> r (pq.execute());

	for(result<Hash>::iterator itr (r.begin()); itr != r.end(); ++itr) {
		hash = *itr;
		break;
	}

	t.commit();

	return hash;
}

imageHasher::db::table::Hash Database::get_hash(u_int64_t phash) {
	Hash hash;
	LOG4CPLUS_DEBUG(logger, "Getting hash for pHash " << phash);

	transaction t(orm_db->begin());

	uint64_t *param;
	odb::prepared_query<Hash> pq = prep_query->get_hash_query(param);
	*param = phash;

	result<Hash> r(pq.execute());

	for (result<Hash>::iterator itr(r.begin()); itr != r.end(); ++itr) {
		hash = *itr;
		break;
	}

	t.commit();

	return hash;
}

imageHasher::db::table::ImageRecord Database::get_imagerecord(fs::path filepath) {
	ImageRecord ir;

	try {
		transaction t(orm_db->begin());

		std::string *p;

		LOG4CPLUS_DEBUG(logger, "Getting imagerecord for path " << filepath);
		odb::prepared_query<ImageRecord> pq = this->prep_query->get_imagerecord_path_query(p);

		*p = filepath.string();

		result<ImageRecord> r(pq.execute());

		for (result<ImageRecord>::iterator itr(r.begin()); itr != r.end(); ++itr) {
			ir = *itr;
			break;
		}

		t.commit();
	} catch (odb::exception *e) {
		LOG4CPLUS_ERROR(logger, "Failed to get ImageRecord for path " << filepath << " : " << e);
	}

	return ir;
}

Hash Database::addHashEntry(std::string sha, u_int64_t pHash) {
	Hash hash(sha,pHash);
	transaction t (orm_db->begin());
		orm_db->persist(hash);
	t.commit();

	return hash;
}

int Database::prune_hash_table() {
	//TODO Replace with ODB ORM queries

	transaction t (orm_db->begin());

	LOG4CPLUS_INFO(logger, "Pruning hash table...");
	unsigned long long query_result = orm_db->execute(prune_hash_table_query);

	t.commit();

	LOG4CPLUS_INFO(logger, "Pruned " << query_result << " record(s) from the hash table.");

	return (int)query_result;
}
<commit_msg>Fixed hash table prune count incorrect<commit_after>/*
 * ImageHasher - C++ version of the Hashing module from SimilarImage
 * Copyright (C) 2014  Nicholas Wright
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

/*
 * Database.cpp
 *
 *  Created on: 21 Aug 2013
 *      Author: nicholas
 */

#include "../include/Database.hpp"
#include <string>

#include <odb/transaction.hxx>
#include <odb/result.hxx>
#include <odb/schema-catalog.hxx>
#include <odb/sqlite/exceptions.hxx>
#include <odb/exceptions.hxx>

#include "table/ImageRecord.hpp"
#include "table/ImageRecord-odb.hxx"

#include "table/FilterRecord.hpp"
#include "table/FilterRecord-odb.hxx"

const char *dbName = "imageHasher.db";

const char *prune_hash_table_query = "DELETE FROM hash WHERE hash_id IN (SELECT hash_id FROM (SELECT imagerecord.hash, hash.hash_id FROM hash LEFT OUTER JOIN imagerecord  ON hash = hash_id) WHERE hash IS null);";

using namespace odb;
using namespace imageHasher::db::table;

Database::Database(const char* dbPath) {
	dbName = dbPath;
	init();
}

Database::Database() {
	init();
}

Database::~Database() {
	shutdown();
	delete this->prep_query;
	delete orm_db;
}

int Database::flush() {
	int drainCount = 0;

	LOG4CPLUS_INFO(logger, "Flushing lists...");

	drainCount = drain();
	drainCount += drain();

	return drainCount;
}

void Database::shutdown() {
	if (running) {
		LOG4CPLUS_INFO(logger, "Shutting down...");
		running = false;
		LOG4CPLUS_INFO(logger, "Waiting for db worker to finish...");
		workerThread->interrupt();
		workerThread->join();
		delete(workerThread);
		LOG4CPLUS_INFO(logger, "Closing database...");
	}
}

void Database::exec(const char* command) {
	orm_db->execute(command);
}

void Database::init() {
	boost::mutex::scoped_lock lock(dbMutex);
	this->currentList = &dataA;
	this->recordsWritten = 0;
	this->running = true;
	logger = log4cplus::Logger::getInstance(LOG4CPLUS_TEXT("Database"));

	setupDatabase();
	prepareStatements();

	workerThread = new boost::thread(&Database::doWork, this);
}

bool Database::is_db_initialised() {
	Hash hash;
	bool exists = true;
	transaction t(orm_db->begin());
	try {
		exists = orm_db->find(1, hash);
	} catch (odb::sqlite::database_exception& e) {
		exists = false;
	}
	t.commit();
	return exists;
}

void Database::initialise_db() {
	transaction t(orm_db->begin());
	odb::schema_catalog::create_schema(*orm_db, "", false);
	t.commit();
	addHashEntry("", 0);
}

void Database::setupDatabase() {
	orm_db = new sqlite::database(dbName,SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE);

	LOG4CPLUS_INFO(logger, "Setting up database " << dbName);

	bool init_done = is_db_initialised();

	if(! init_done) {
		initialise_db();
	}

	transaction t_pragma (orm_db->begin());
		orm_db->execute("PRAGMA page_size = 4096;");
		orm_db->execute("PRAGMA cache_size=10000;");
		orm_db->execute("PRAGMA locking_mode=EXCLUSIVE;");
		orm_db->execute("PRAGMA temp_store = MEMORY;");
		orm_db->execute("PRAGMA journal_mode=MEMORY;");
	t_pragma.commit();

	// perform out of transaction execution
	connection_ptr c (orm_db->connection());
	c->execute("PRAGMA synchronous=NORMAL;");
}

void Database::add(db_data data) {
	boost::mutex::scoped_lock lock(flipMutex);
	currentList->push_back(data);
}

void Database::flipLists() {
	boost::mutex::scoped_lock lock(flipMutex);

	if(currentList == &dataA) {
		currentList = &dataB;
	}else{
		currentList = &dataA;
	}
}

int Database::drain() {
	boost::mutex::scoped_lock lock(dbMutex);
	std::list<db_data>* workList;
	int drainCount = 0;

	workList = currentList;
	flipLists();

	for(std::list<db_data>::iterator ite = workList->begin(); ite != workList->end(); ++ite) {
		addToBatch(*ite);
		recordsWritten++;
		drainCount++;
	}

	workList->clear();
	return drainCount;
}

bool Database::entryExists(fs::path filePath) {
	LOG4CPLUS_DEBUG(logger, "Looking for file " << filePath);

	ImageRecord ir = get_imagerecord(filePath);

	return ir.is_valid();
}

bool Database::entryExists(db_data data) {
	return entryExists(data.filePath);
}

std::list<fs::path> Database::getFilesWithPath(fs::path directoryPath) {
	std::list<fs::path> filePaths;
	std::string path_query(directoryPath.string());
	path_query += "%";

	LOG4CPLUS_INFO(logger, "Looking for files with path " << directoryPath);

	transaction t (orm_db->begin());
	std::string *param;

	odb::prepared_query<ImageRecord> pq = prep_query->get_files_with_path_query(param);

	*param = path_query;

	odb::result<ImageRecord> r(pq.execute());

	for(result<ImageRecord>::iterator itr (r.begin()); itr != r.end(); ++itr) {
		filePaths.push_back(fs::path(itr->getPath()));
	}

	t.commit();

	LOG4CPLUS_INFO(logger, "Found  " << filePaths.size() << " records for path " << directoryPath);
	return filePaths;
}

void Database::prunePath(std::list<fs::path> filePaths) {
	LOG4CPLUS_INFO(logger, "Pruning " << filePaths.size() << " path(s) from the database.");

	for(std::list<fs::path>::iterator ite = filePaths.begin(); ite != filePaths.end(); ++ite){
		ImageRecord ir = get_imagerecord(*ite);

		transaction t (orm_db->begin());

		if(ir.is_valid()) {
			orm_db->erase(ir);
		}

		t.commit();
	}
}

void Database::addToBatch(db_data data) {
	switch (data.status) {
	case OK:
		add_record(data);
		break;

	case INVALID:
		add_invalid(data);
		break;

	case FILTER:
		add_filter(data);
		break;

	default:
		LOG4CPLUS_ERROR(logger, "Unhandled state encountered");
		throw "Unhandled state encountered";
		break;
	}
}

void Database::add_record(db_data data) {
	Hash hash = get_hash(data.sha256);

	if (!hash.is_valid()) {
		hash = addHashEntry(data.sha256, data.pHash);
	}

	if(entryExists(data)) {
		LOG4CPLUS_DEBUG(logger, "Entry for " << data.filePath << " already exists, discarding...");
		recordsWritten--;
		return;
	}

	try{
	transaction t(orm_db->begin());

	ImageRecord ir = ImageRecord(data.filePath.string(), &hash);
	orm_db->persist(ir);

	t.commit();
	} catch (const odb::exception &e) {
		LOG4CPLUS_ERROR(logger, "Failed to add ImageRecord for path " << data.filePath << " : " << e.what());
	}
}

int Database::add_path_placeholder(std::string path) {

	transaction t (orm_db->begin());

	ImageRecord ir (path,NULL);
	int id = orm_db->persist(ir);

	t.commit();

	return id;
}

void Database::add_invalid(db_data data) {
	LOG4CPLUS_DEBUG(logger, "File with path " << data.filePath << " is invalid");
	recordsWritten--;
	Hash hash = get_hash("");
	transaction t(orm_db->begin());
	ImageRecord ir = ImageRecord(data.filePath.string(), &hash);
	orm_db->persist(ir);

	t.commit();
}

void Database::add_filter(db_data data) {
	LOG4CPLUS_DEBUG(logger, "Adding filter record for pHash" << data.pHash << " with reason " << data.reason);

	imageHasher::db::table::FilterRecord fr(data.pHash, data.reason);

	transaction t (orm_db->begin());

	orm_db->persist(fr);

	t.commit();
}

void Database::prepareStatements() {
	LOG4CPLUS_INFO(logger, "Creating prepared statements...");
	this->prep_query = new imageHasher::db::PreparedQuery(this->orm_db);
}

void Database::doWork() {
	while(running) {
		try{
			boost::this_thread::sleep_for(boost::chrono::seconds(3));
		}catch(boost::thread_interrupted&) {
			LOG4CPLUS_INFO(logger,"DB thread interrupted");
		}

		if(currentList->size() > 1000) {
			int drainCount = drain();
			LOG4CPLUS_INFO(logger, drainCount << " records processed, Total: " << recordsWritten);
		}
	}

	// make sure both lists are committed
	int drainCount = flush();
	LOG4CPLUS_INFO(logger, drainCount << " records processed, Total: " << recordsWritten);
}

unsigned int Database::getRecordsWritten() {
	return recordsWritten;
}

std::string Database::getSHA(fs::path filepath) {
	std::string sha = "";
	LOG4CPLUS_DEBUG(logger, "Getting SHA for path " << filepath);

	ImageRecord ir = get_imagerecord(filepath);

	if(ir.is_valid()) {
		Hash hash = ir.get_hash();
		sha = hash.get_sha256();
	}

	return sha;
}

bool Database::sha_exists(std::string sha) {
	Hash hash = get_hash(sha);
	return hash.is_valid();
}

int64_t Database::getPhash(fs::path filepath) {
	LOG4CPLUS_DEBUG(logger, "Getting pHash for path " << filepath);

	ImageRecord ir = get_imagerecord(filepath);

	if(ir.is_valid()) {
		return ir.get_hash().get_pHash();
	}

	return -1;
}

imageHasher::db::table::Hash Database::get_hash(std::string sha) {
	Hash hash;
	LOG4CPLUS_DEBUG(logger, "Getting hash for sha " << sha);

	transaction t (orm_db->begin());

	std::string *param;

	odb::prepared_query<Hash> pq = prep_query->get_hash_query(param);

	*param = sha;

	result<Hash> r (pq.execute());

	for(result<Hash>::iterator itr (r.begin()); itr != r.end(); ++itr) {
		hash = *itr;
		break;
	}

	t.commit();

	return hash;
}

imageHasher::db::table::Hash Database::get_hash(u_int64_t phash) {
	Hash hash;
	LOG4CPLUS_DEBUG(logger, "Getting hash for pHash " << phash);

	transaction t(orm_db->begin());

	uint64_t *param;
	odb::prepared_query<Hash> pq = prep_query->get_hash_query(param);
	*param = phash;

	result<Hash> r(pq.execute());

	for (result<Hash>::iterator itr(r.begin()); itr != r.end(); ++itr) {
		hash = *itr;
		break;
	}

	t.commit();

	return hash;
}

imageHasher::db::table::ImageRecord Database::get_imagerecord(fs::path filepath) {
	ImageRecord ir;

	try {
		transaction t(orm_db->begin());

		std::string *p;

		LOG4CPLUS_DEBUG(logger, "Getting imagerecord for path " << filepath);
		odb::prepared_query<ImageRecord> pq = this->prep_query->get_imagerecord_path_query(p);

		*p = filepath.string();

		result<ImageRecord> r(pq.execute());

		for (result<ImageRecord>::iterator itr(r.begin()); itr != r.end(); ++itr) {
			ir = *itr;
			break;
		}

		t.commit();
	} catch (odb::exception *e) {
		LOG4CPLUS_ERROR(logger, "Failed to get ImageRecord for path " << filepath << " : " << e);
	}

	return ir;
}

Hash Database::addHashEntry(std::string sha, u_int64_t pHash) {
	Hash hash(sha,pHash);
	transaction t (orm_db->begin());
		orm_db->persist(hash);
	t.commit();

	return hash;
}

int Database::prune_hash_table() {
	//TODO Replace with ODB ORM queries

	transaction t (orm_db->begin());

	LOG4CPLUS_INFO(logger, "Pruning hash table...");
	unsigned long long query_result = orm_db->execute(prune_hash_table_query);

	t.commit();

	query_result -= 1;

	LOG4CPLUS_INFO(logger, "Pruned " << query_result << " record(s) from the hash table.");

	return (int)query_result;
}
<|endoftext|>


<commit_before>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

 /*

Workflow:

$ ./native/redex/redex.py -u <APK>
$ buck run  //native/redex:redex-tool -- dex-sql-dump  \
      --apkdir <APKDIR> --dexendir <DEXEN_DIR> \
      --jars <ANDROID_JAR> --proguard-map <RENAME_MAP> \
      --output dex.sql
$ sqlite3 dex.db < dex.sql
$ sqlite3 dex.db "SELECT COUNT(*) FROM dex;"   # verify sane-looking value
$ ./native/redex/tools/redex-tool/DexSqlQuery.py dex.db
<..enter queries..>

*/

#include "ControlFlow.h"
#include "DexOutput.h"
#include "Resolver.h"
#include "Show.h"
#include "Tool.h"
#include "Transform.h"
#include "ClassHierarchy.h"
#include "Walkers.h"

#include <boost/algorithm/string/replace.hpp>
#include <queue>
#include <vector>
#include <unordered_map>

namespace {

static std::unordered_map<DexClass*, int> class_ids;
static std::unordered_map<DexMethod*, int> method_ids;
static std::unordered_map<DexField*, int> field_ids;
static std::unordered_map<DexString*, int> string_ids;

void dump_field_refs(FILE* fdout, DexField* field, int field_id) {
  static int next_string_ref = 0;
  auto* static_value = field->get_static_value();
  if (!static_value || (static_value->evtype() != DEVT_STRING)) return;
  auto* static_string_value = static_cast<DexEncodedValueString*>(static_value);
  auto string_id = string_ids[static_string_value->string()];
  fprintf(
    fdout,
    "INSERT INTO field_string_refs VALUES (%d, %d, %d);\n",
    next_string_ref++,
    field_id,
    string_id);
}

void dump_method_refs(FILE* fdout, DexMethod* method, int method_id) {
  auto code = method->get_code();
  if (!code) return;

  static int next_string_ref = 0;
  static int next_class_ref = 0;
  static int next_field_ref = 0;
  static int next_method_ref = 0;

  for (auto& mie : InstructionIterable(code)) {
    auto insn = mie.insn;
    if (insn->has_string()) {
      if (string_ids.count(insn->get_string())) {
        auto string_id = string_ids[insn->get_string()];
        fprintf(
          fdout,
          "INSERT INTO method_string_refs VALUES (%d, %d, %d, %d);\n",
          next_string_ref++,
          method_id,
          string_id,
          insn->opcode());
      }
    }
    if (insn->has_type()) {
      auto cls = type_class(insn->get_type());
      if (cls && class_ids.count(cls)) {
        auto class_id = class_ids[cls];
        fprintf(
          fdout,
          "INSERT INTO method_class_refs VALUES (%d, %d, %d, %d);\n",
          next_class_ref++,
          method_id,
          class_id,
          insn->opcode());
      }
    }
    if (insn->has_field()) {
      auto field = insn->get_field();
      if (!field->is_def()) {
        field = resolve_field(field);
        if (!field) {
          field = insn->get_field();
        }
      }
      if (field_ids.count(field)) {
        auto field_id = field_ids[insn->get_field()];
        fprintf(
          fdout,
          "INSERT INTO method_field_refs VALUES (%d, %d, %d, %d);\n",
          next_field_ref++,
          method_id,
          field_id,
          insn->opcode());
      }
    }
    if (insn->has_method()) {
      auto meth = insn->get_method();
      if (!meth->is_def()) {
        meth = resolve_method(meth, MethodSearch::Any);
        if (!meth) {
          meth = insn->get_method();
        }
      }
      if (method_ids.count(meth)) {
        auto method_ref_id = method_ids[insn->get_method()];
        fprintf(
          fdout,
          "INSERT INTO method_method_refs VALUES (%d, %d, %d, %d);\n",
          next_method_ref++,
          method_id,
          method_ref_id,
          insn->opcode());
      }
    }
  }
}

void dump_class(FILE* fdout, const char* dex_id, DexClass* cls, int class_id) {
  // TODO: annotations?
  // TODO: inheritance?
  // TODO: string usage
  // TODO: size estimate
  auto deobfuscated_name = cls->get_deobfuscated_name();
  fprintf(
    fdout,
    "INSERT INTO classes VALUES (%d,'%s','%s','%s',%u);\n",
    class_id,
    dex_id,
    deobfuscated_name.c_str(),
    cls->get_name()->c_str(),
    cls->get_access()
  );
}

void dump_field(FILE* fdout, int class_id, DexField* field, int field_id) {
  // TODO: more fixup here on this crapped up name/signature
  // TODO: break down signature
  // TODO: annotations?
  // TODO: string usage (encoded_value for static fields)
  auto deobfuscated_name = field->get_deobfuscated_name();
  auto field_name = strchr(deobfuscated_name.c_str(), ';');
  fprintf(
    fdout,
    "INSERT INTO fields VALUES(%d, %d, '%s', '%s', %u);\n",
    field_id,
    class_id,
    field_name,
    field->get_name()->c_str(),
    field->get_access()
  );
}

void dump_method(FILE* fdout, int class_id, DexMethod* method, int method_id) {
  // TODO: more fixup here on this crapped up name/signature
  // TODO: break down signature
  // TODO: throws?
  // TODO: annotations?
  // TODO: string usage
  // TODO: size estimate
  auto deobfuscated_name = method->get_deobfuscated_name();
  auto method_name = strchr(deobfuscated_name.c_str(), ';');
  fprintf(
    fdout,
    "INSERT INTO methods VALUES (%d,%d,'%s','%s',%d,%lu);\n",
    method_id,
    class_id,
    method_name,
    method->get_name()->c_str(),
    method->get_access(),
    method->get_code() ? method->get_code()->sum_opcode_sizes() : 0
  );
}

void dump_sql(FILE* fdout, DexStoresVector& stores, ProguardMap& pg_map) {
  fprintf(
    fdout,
R"___(
DROP TABLE IF EXISTS field_string_refs;
DROP TABLE IF EXISTS method_string_refs;
DROP TABLE IF EXISTS method_field_refs;
DROP TABLE IF EXISTS method_method_refs;
DROP TABLE IF EXISTS method_class_refs;
DROP TABLE IF EXISTS methods;
DROP TABLE IF EXISTS strings;
DROP TABLE IF EXISTS fields;
DROP TABLE IF EXISTS is_a;
DROP TABLE IF EXISTS methods;
DROP TABLE IF EXISTS classes;
CREATE TABLE classes (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  dex TEXT NOT NULL, -- dex identifiers look like "<store>/<dex_id>"
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL
);
CREATE TABLE methods (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL,
  code_size INTEGER NOT NULL
);
CREATE TABLE is_a (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  is_a_class_id INTEGER -- fk:classes.id
);
CREATE TABLE strings (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  text TEXT NOT NULL
);
CREATE TABLE fields (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL
);
CREATE TABLE field_string_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  field_id INTEGER NOT NULL, -- fk:fields.id
  ref_string_id INTEGER NOT NULL -- fk:strings.id
);
CREATE TABLE method_class_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_class_id INTEGER NOT NULL, -- fk:classes.id
  opcode INTEGER NOT NULL
);
CREATE TABLE method_method_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_method_id INTEGER NOT NULL, -- fk:methods.id
  opcode INTEGER NOT NULL
);
CREATE TABLE method_field_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_field_id INTEGER NOT NULL, -- fk:fields.id
  opcode INTEGER NOT NULL
);
CREATE TABLE method_string_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_string_id INTEGER NOT NULL, -- fk:strings.id
  opcode INTEGER NOT NULL
);
)___"
  );
  int next_class_id = 0;
  int next_method_id = 0;
  int next_field_id = 0;
  int next_string_id = 0;

  // Dump all dex items
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& store : stores) {
    auto store_name = store.get_name();
    auto& dexen = store.get_dexen();
    apply_deobfuscated_names(dexen, pg_map);
    for (size_t dex_idx = 0 ; dex_idx < dexen.size() ; ++dex_idx) {
      auto& dex = dexen[dex_idx];
      GatheredTypes gtypes(&dex);
      auto strings = gtypes.get_cls_order_dexstring_emitlist();
      for (auto dexstr : strings) {
        int id = next_string_id++;
        string_ids[dexstr] = id;
        // Escape string before inserting. ' -> ''
        std::string esc(dexstr->c_str());
        boost::replace_all(esc, "'", "''");
        fprintf(fdout, "INSERT INTO strings VALUES(%d, '%s');\n", id, esc.c_str());
      }
      std::string dex_id_str(store_name + "/" + std::to_string(dex_idx));
      const char* dex_id = dex_id_str.c_str();
      for (const auto& cls : dex) {
        int class_id = next_class_id++;
        dump_class(fdout, dex_id, cls, class_id);
        class_ids[cls] = class_id;
        for (auto field : cls->get_ifields()) {
          int field_id = next_field_id++;
          field_ids[field] = field_id;
          dump_field(fdout, class_id, field, field_id);
        }
        for (auto field : cls->get_sfields()) {
          int field_id = next_field_id++;
          field_ids[field] = field_id;
          dump_field(fdout, class_id, field, field_id);
        }
        for (const auto& meth : cls->get_dmethods()) {
          int meth_id = next_method_id++;
          method_ids[meth] = meth_id;
          dump_method(fdout, class_id, meth, meth_id);
        }
        for (auto& meth : cls->get_vmethods()) {
          int meth_id = next_method_id++;
          method_ids[meth] = meth_id;
          dump_method(fdout, class_id, meth, meth_id);
        }
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");

  // Dump references
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& store : stores) {
    auto& dexen = store.get_dexen();
    for (size_t dex_idx = 0 ; dex_idx < dexen.size() ; ++dex_idx) {
      auto& dex = dexen[dex_idx];
      for (const auto& cls : dex) {
        for (const auto& meth : cls->get_dmethods()) {
          int meth_id = method_ids[meth];
          dump_method_refs(fdout, meth, meth_id);
        }
        for (auto& meth : cls->get_vmethods()) {
          int meth_id = method_ids[meth];
          dump_method_refs(fdout, meth, meth_id);
        }
        for (const auto& field : cls->get_sfields()) {
          int field_id = field_ids[field];
          dump_field_refs(fdout, field, field_id);
        }
        for (const auto& field : cls->get_ifields()) {
          int field_id = field_ids[field];
          dump_field_refs(fdout, field, field_id);
        }
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");

  // Dump hierarchy
  auto scope = build_class_scope(stores);
  ClassHierarchy ch = build_type_hierarchy(scope);
  int next_is_a_id = 0;
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& cls : scope) {
    TypeSet results;
    get_all_children_or_implementors(ch, scope, cls, results);
    for(auto type : results) {
      auto type_cls = type_class(type);
      if (type_cls) {
        fprintf(
          fdout,
          "INSERT INTO is_a VALUES(%d, %d, %d);\n",
          next_is_a_id++,
          class_ids[type_cls],
          class_ids[cls]
        );
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");
}

class DexSqlDump : public Tool {
 public:
  DexSqlDump() : Tool("dex-sql-dump", "dump an apk to a sql insertion script") {}

  void add_options(po::options_description& options) const override {
    add_standard_options(options);
    options.add_options()
      ("proguard-map,p",
       po::value<std::string>()->value_name("redex-rename-map.txt"),
       "path to a rename map")
      ("output,o",
       po::value<std::string>()->value_name("dex.sql"),
       "path to output sql dump file (defaults to stdout)")
    ;
  }

  void run(const po::variables_map& options) override {
    auto stores = init(
      options["jars"].as<std::string>(),
      options["apkdir"].as<std::string>(),
      options["dexendir"].as<std::string>());
    ProguardMap pgmap(options.count("proguard-map") ?
      options["proguard-map"].as<std::string>() : "/dev/null");
    auto filename = options["output"].as<std::string>().c_str();
    FILE* fdout = options.count("output") ?
      fopen(filename, "w") : stdout;
    if (!fdout) {
      fprintf(stderr, "Could not open %s for writing; terminating\n", filename);
      exit(EXIT_FAILURE);
    }
    dump_sql(fdout, stores, pgmap);
    fclose(fdout);
  }
};

static DexSqlDump s_tool;

}
<commit_msg>add --table-prefix to dexsql<commit_after>/**
 * Copyright (c) 2016-present, Facebook, Inc.
 * All rights reserved.
 *
 * This source code is licensed under the BSD-style license found in the
 * LICENSE file in the root directory of this source tree. An additional grant
 * of patent rights can be found in the PATENTS file in the same directory.
 */

 /*

Workflow:

$ ./native/redex/redex.py -u <APK>
$ buck run  //native/redex:redex-tool -- dex-sql-dump  \
      --apkdir <APKDIR> --dexendir <DEXEN_DIR> \
      --jars <ANDROID_JAR> --proguard-map <RENAME_MAP> \
      --output dex.sql
$ sqlite3 dex.db < dex.sql
$ sqlite3 dex.db "SELECT COUNT(*) FROM dex;"   # verify sane-looking value
$ ./native/redex/tools/redex-tool/DexSqlQuery.py dex.db
<..enter queries..>

*/

#include "ControlFlow.h"
#include "DexOutput.h"
#include "Resolver.h"
#include "Show.h"
#include "Tool.h"
#include "Transform.h"
#include "ClassHierarchy.h"
#include "Walkers.h"

#include <boost/algorithm/string/replace.hpp>
#include <queue>
#include <vector>
#include <unordered_map>

namespace {

static std::unordered_map<DexClass*, int> class_ids;
static std::unordered_map<DexMethod*, int> method_ids;
static std::unordered_map<DexField*, int> field_ids;
static std::unordered_map<DexString*, int> string_ids;

void dump_field_refs(FILE* fdout, const char* prefix, DexField* field, int field_id) {
  static int next_string_ref = 0;
  auto* static_value = field->get_static_value();
  if (!static_value || (static_value->evtype() != DEVT_STRING)) return;
  auto* static_string_value = static_cast<DexEncodedValueString*>(static_value);
  auto string_id = string_ids[static_string_value->string()];
  fprintf(
    fdout,
    "INSERT INTO %sfield_string_refs VALUES (%d, %d, %d);\n",
    prefix,
    next_string_ref++,
    field_id,
    string_id);
}

void dump_method_refs(FILE* fdout, const char* prefix, DexMethod* method, int method_id) {
  auto code = method->get_code();
  if (!code) return;

  static int next_string_ref = 0;
  static int next_class_ref = 0;
  static int next_field_ref = 0;
  static int next_method_ref = 0;

  for (auto& mie : InstructionIterable(code)) {
    auto insn = mie.insn;
    if (insn->has_string()) {
      if (string_ids.count(insn->get_string())) {
        auto string_id = string_ids[insn->get_string()];
        fprintf(
          fdout,
          "INSERT INTO %smethod_string_refs VALUES (%d, %d, %d, %d);\n",
          prefix,
          next_string_ref++,
          method_id,
          string_id,
          insn->opcode());
      }
    }
    if (insn->has_type()) {
      auto cls = type_class(insn->get_type());
      if (cls && class_ids.count(cls)) {
        auto class_id = class_ids[cls];
        fprintf(
          fdout,
          "INSERT INTO %smethod_class_refs VALUES (%d, %d, %d, %d);\n",
          prefix,
          next_class_ref++,
          method_id,
          class_id,
          insn->opcode());
      }
    }
    if (insn->has_field()) {
      auto field = insn->get_field();
      if (!field->is_def()) {
        field = resolve_field(field);
        if (!field) {
          field = insn->get_field();
        }
      }
      if (field_ids.count(field)) {
        auto field_id = field_ids[insn->get_field()];
        fprintf(
          fdout,
          "INSERT INTO %smethod_field_refs VALUES (%d, %d, %d, %d);\n",
          prefix,
          next_field_ref++,
          method_id,
          field_id,
          insn->opcode());
      }
    }
    if (insn->has_method()) {
      auto meth = insn->get_method();
      if (!meth->is_def()) {
        meth = resolve_method(meth, MethodSearch::Any);
        if (!meth) {
          meth = insn->get_method();
        }
      }
      if (method_ids.count(meth)) {
        auto method_ref_id = method_ids[insn->get_method()];
        fprintf(
          fdout,
          "INSERT INTO %smethod_method_refs VALUES (%d, %d, %d, %d);\n",
          prefix,
          next_method_ref++,
          method_id,
          method_ref_id,
          insn->opcode());
      }
    }
  }
}

void dump_class(FILE* fdout, const char* prefix, const char* dex_id, DexClass* cls, int class_id) {
  // TODO: annotations?
  // TODO: inheritance?
  // TODO: string usage
  // TODO: size estimate
  auto deobfuscated_name = cls->get_deobfuscated_name();
  fprintf(
    fdout,
    "INSERT INTO %sclasses VALUES (%d,'%s','%s','%s',%u);\n",
    prefix,
    class_id,
    dex_id,
    deobfuscated_name.c_str(),
    cls->get_name()->c_str(),
    cls->get_access()
  );
}

void dump_field(FILE* fdout, const char* prefix, int class_id, DexField* field, int field_id) {
  // TODO: more fixup here on this crapped up name/signature
  // TODO: break down signature
  // TODO: annotations?
  // TODO: string usage (encoded_value for static fields)
  auto deobfuscated_name = field->get_deobfuscated_name();
  auto field_name = strchr(deobfuscated_name.c_str(), ';');
  fprintf(
    fdout,
    "INSERT INTO %sfields VALUES(%d, %d, '%s', '%s', %u);\n",
    prefix,
    field_id,
    class_id,
    field_name,
    field->get_name()->c_str(),
    field->get_access()
  );
}

void dump_method(FILE* fdout, const char* prefix, int class_id, DexMethod* method, int method_id) {
  // TODO: more fixup here on this crapped up name/signature
  // TODO: break down signature
  // TODO: throws?
  // TODO: annotations?
  // TODO: string usage
  // TODO: size estimate
  auto deobfuscated_name = method->get_deobfuscated_name();
  auto method_name = strchr(deobfuscated_name.c_str(), ';');
  fprintf(
    fdout,
    "INSERT INTO %smethods VALUES (%d,%d,'%s','%s',%d,%lu);\n",
    prefix,
    method_id,
    class_id,
    method_name,
    method->get_name()->c_str(),
    method->get_access(),
    method->get_code() ? method->get_code()->sum_opcode_sizes() : 0
  );
}

void dump_sql(
  FILE* fdout,
  DexStoresVector& stores,
  ProguardMap& pg_map,
  const char* prefix) {
  fprintf(
    fdout,
R"___(
DROP TABLE IF EXISTS %1$sfield_string_refs;
DROP TABLE IF EXISTS %1$smethod_string_refs;
DROP TABLE IF EXISTS %1$smethod_field_refs;
DROP TABLE IF EXISTS %1$smethod_method_refs;
DROP TABLE IF EXISTS %1$smethod_class_refs;
DROP TABLE IF EXISTS %1$sstrings;
DROP TABLE IF EXISTS %1$sfields;
DROP TABLE IF EXISTS %1$sis_a;
DROP TABLE IF EXISTS %1$smethods;
DROP TABLE IF EXISTS %1$sclasses;
CREATE TABLE %1$sclasses (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  dex TEXT NOT NULL, -- dex identifiers look like "<store>/<dex_id>"
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL
);
CREATE TABLE %1$smethods (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL,
  code_size INTEGER NOT NULL
);
CREATE TABLE %1$sis_a (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  is_a_class_id INTEGER -- fk:classes.id
);
CREATE TABLE %1$sstrings (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  text TEXT NOT NULL
);
CREATE TABLE %1$sfields (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  class_id INTEGER, -- fk:classes.id
  name TEXT NOT NULL,
  obfuscated_name TEXT NOT NULL,
  access INTEGER NOT NULL
);
CREATE TABLE %1$sfield_string_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  field_id INTEGER NOT NULL, -- fk:fields.id
  ref_string_id INTEGER NOT NULL -- fk:strings.id
);
CREATE TABLE %1$smethod_class_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_class_id INTEGER NOT NULL, -- fk:classes.id
  opcode INTEGER NOT NULL
);
CREATE TABLE %1$smethod_method_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_method_id INTEGER NOT NULL, -- fk:methods.id
  opcode INTEGER NOT NULL
);
CREATE TABLE %1$smethod_field_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_field_id INTEGER NOT NULL, -- fk:fields.id
  opcode INTEGER NOT NULL
);
CREATE TABLE %1$smethod_string_refs (
  id INTEGER PRIMARY KEY AUTOINCREMENT,
  method_id INTEGER, -- fk:methods.id
  ref_string_id INTEGER NOT NULL, -- fk:strings.id
  opcode INTEGER NOT NULL
);
)___",
    prefix
  );
  int next_class_id = 0;
  int next_method_id = 0;
  int next_field_id = 0;
  int next_string_id = 0;

  // Dump all dex items
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& store : stores) {
    auto store_name = store.get_name();
    auto& dexen = store.get_dexen();
    apply_deobfuscated_names(dexen, pg_map);
    for (size_t dex_idx = 0 ; dex_idx < dexen.size() ; ++dex_idx) {
      auto& dex = dexen[dex_idx];
      GatheredTypes gtypes(&dex);
      auto strings = gtypes.get_cls_order_dexstring_emitlist();
      for (auto dexstr : strings) {
        int id = next_string_id++;
        string_ids[dexstr] = id;
        // Escape string before inserting. ' -> ''
        std::string esc(dexstr->c_str());
        boost::replace_all(esc, "'", "''");
        fprintf(fdout, "INSERT INTO %sstrings VALUES(%d, '%s');\n", prefix, id, esc.c_str());
      }
      std::string dex_id_str(store_name + "/" + std::to_string(dex_idx));
      const char* dex_id = dex_id_str.c_str();
      for (const auto& cls : dex) {
        int class_id = next_class_id++;
        dump_class(fdout, prefix, dex_id, cls, class_id);
        class_ids[cls] = class_id;
        for (auto field : cls->get_ifields()) {
          int field_id = next_field_id++;
          field_ids[field] = field_id;
          dump_field(fdout, prefix, class_id, field, field_id);
        }
        for (auto field : cls->get_sfields()) {
          int field_id = next_field_id++;
          field_ids[field] = field_id;
          dump_field(fdout, prefix, class_id, field, field_id);
        }
        for (const auto& meth : cls->get_dmethods()) {
          int meth_id = next_method_id++;
          method_ids[meth] = meth_id;
          dump_method(fdout, prefix, class_id, meth, meth_id);
        }
        for (auto& meth : cls->get_vmethods()) {
          int meth_id = next_method_id++;
          method_ids[meth] = meth_id;
          dump_method(fdout, prefix, class_id, meth, meth_id);
        }
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");

  // Dump references
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& store : stores) {
    auto& dexen = store.get_dexen();
    for (size_t dex_idx = 0 ; dex_idx < dexen.size() ; ++dex_idx) {
      auto& dex = dexen[dex_idx];
      for (const auto& cls : dex) {
        for (const auto& meth : cls->get_dmethods()) {
          int meth_id = method_ids[meth];
          dump_method_refs(fdout, prefix, meth, meth_id);
        }
        for (auto& meth : cls->get_vmethods()) {
          int meth_id = method_ids[meth];
          dump_method_refs(fdout, prefix, meth, meth_id);
        }
        for (const auto& field : cls->get_sfields()) {
          int field_id = field_ids[field];
          dump_field_refs(fdout, prefix, field, field_id);
        }
        for (const auto& field : cls->get_ifields()) {
          int field_id = field_ids[field];
          dump_field_refs(fdout, prefix, field, field_id);
        }
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");

  // Dump hierarchy
  auto scope = build_class_scope(stores);
  ClassHierarchy ch = build_type_hierarchy(scope);
  int next_is_a_id = 0;
  fprintf(fdout, "BEGIN TRANSACTION;\n");
  for (auto& cls : scope) {
    TypeSet results;
    get_all_children_or_implementors(ch, scope, cls, results);
    for(auto type : results) {
      auto type_cls = type_class(type);
      if (type_cls) {
        fprintf(
          fdout,
          "INSERT INTO %sis_a VALUES(%d, %d, %d);\n",
          prefix,
          next_is_a_id++,
          class_ids[type_cls],
          class_ids[cls]
        );
      }
    }
  }
  fprintf(fdout, "END TRANSACTION;\n");
}

class DexSqlDump : public Tool {
 public:
  DexSqlDump() : Tool("dex-sql-dump", "dump an apk to a sql insertion script") {}

  void add_options(po::options_description& options) const override {
    add_standard_options(options);
    options.add_options()
      ("proguard-map,p",
       po::value<std::string>()->value_name("redex-rename-map.txt"),
       "path to a rename map")
      ("output,o",
       po::value<std::string>()->value_name("dex.sql"),
       "path to output sql dump file (defaults to stdout)")
      ("table-prefix,t",
       po::value<std::string>()->value_name("pre_"),
       "prefix to use on all table names")
    ;
  }

  void run(const po::variables_map& options) override {
    auto stores = init(
      options["jars"].as<std::string>(),
      options["apkdir"].as<std::string>(),
      options["dexendir"].as<std::string>());
    ProguardMap pgmap(options.count("proguard-map") ?
      options["proguard-map"].as<std::string>() : "/dev/null");
    auto filename = options["output"].as<std::string>().c_str();
    FILE* fdout = options.count("output") ?
      fopen(filename, "w") : stdout;
    std::string prefix = options.count("table-prefix") ?
      options["table-prefix"].as<std::string>() : "";
    if (!fdout) {
      fprintf(stderr, "Could not open %s for writing; terminating\n", filename);
      exit(EXIT_FAILURE);
    }
    auto* pfx_cstr = prefix.c_str();
    dump_sql(fdout, stores, pgmap, pfx_cstr);
    fclose(fdout);
  }
};

static DexSqlDump s_tool;

}
<|endoftext|>