<commit_before>/*
 * empsave.C
 *
 * .bgeo to .emp converter
 *
 * Copyright (c) 2010 Van Aarde Krynauw.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "geo2emp.h"

#include <GEO/GEO_AttributeHandle.h>
#include <GEO/GEO_TriMesh.h>

#include <NgEmp.h>
#include <NgString.h>

/**************************************************************************************************/

Geo2Emp::ErrorCode Geo2Emp::saveMeshShape(Ng::Body*& pMeshBody)
{
	if (!_gdpOut)
	{
		//If we don't have a GDP for writing data into Houdini, return error.
		return EC_NULL_READ_GDP;
	}

	//Create a Naiad mesh body

	//Ng::Body* pMeshBody = new Ng::Body("trimesh");
	pMeshBody = new Ng::Body( _bodyName );
	pMeshBody->match("Mesh");

	//get the mutable shapes for the mesh body.
	Ng::TriangleShape& triShape( pMeshBody->mutableTriangleShape() );
	Ng::PointShape& ptShape( pMeshBody->mutablePointShape() );

	//For the sake of simplicity, copy ALL the points in the GDP across.
	//It will be much slower if we have to filter out all the non-mesh vertices and then remap the vertices to new point numbers.
	const GEO_PointList& ptlist = _gdpOut->points();
	const GEO_PrimList& primlist = _gdpOut->primitives();
	const GEO_Point* ppt;
	const GEO_Primitive* pprim;
	UT_Vector3 pos, v3;
	int numpoints = ptlist.entries();
	int vertcount = 0;
	GEO_AttributeHandle attr_v, attr_N;
	bool attr_v_valid, attr_N_valid;
	Ng::Buffer3f* pNormBuffer = 0;
	Ng::Buffer3f* pVelBuffer = 0;

	attr_v = _gdpOut->getPointAttribute("v");
	attr_N = _gdpOut->getPointAttribute("N");

	attr_v_valid = attr_v.isAttributeValid();
	attr_N_valid = attr_N.isAttributeValid();

	if (attr_v_valid)
	{
		//If the geometry has a velocity attribute, then create one on the mesh shape
		ptShape.createChannel3f( "velocity", em::vec3f(0.0f) );
		pVelBuffer = &(ptShape.mutableBuffer3f("velocity"));
		pVelBuffer->reserve(numpoints);
	}

	if (attr_N_valid)
	{
		//If the geometry has a normal attribute, then create one on the mesh shape
		ptShape.createChannel3f( "normal", em::vec3f(0.0f) );
		pNormBuffer = &(ptShape.mutableBuffer3f("normal"));
		pNormBuffer->reserve(numpoints);
	}

	//Retrieve the positions buffer from the point shape so that we can add all the vertices.
	Ng::Buffer3f& pPosBuf( ptShape.mutableBuffer3f("position") );
	pPosBuf.reserve( numpoints );

	for (int i = 0; i < numpoints; i++)
	{
		ppt = ptlist[i];
		pos = ppt->getPos();
		pPosBuf.push_back( em::vec3f( pos[0], pos[1], pos[2]) );

		//Set the velocity and normal attributes if there are any.
		if (attr_v_valid)
		{
			attr_v.setElement(ppt);
			v3 = attr_v.getV3();
			pVelBuffer->push_back( em::vec3f(v3[0], v3[1], v3[2]) );
		}

		if (attr_N_valid)
		{
			attr_N.setElement(ppt);
			v3 = attr_N.getV3();
			pNormBuffer->push_back( em::vec3f(v3[0], v3[1], v3[2]) );
		}

		//Ponder: Should we export blind data, or keep the attribute exports specific? Maybe blind exports from a SOP could work...
	}

	//Now its time to set the index channel to form faces
	Ng::Buffer3i& indexBuffer = triShape.mutableBuffer3i("index");
	em::vec3i triVec(0,0,0);

	for (pprim = primlist.head(); pprim; pprim = primlist.next(pprim) )
	{
		const GEO_TriMesh* triMesh = dynamic_cast<const GEO_TriMesh*>(pprim);

		if ( triMesh )
		{
			vertcount = triMesh->getVertexCount();

			if (vertcount >= 3)
			{
				//Export the first 3 vertices of this primitive.
				for (int i = 2; i >= 0; i--)
				{
					triVec[i] = triMesh->getVertex(i).getPt()->getNum();
				}
				indexBuffer.push_back( triVec );
			}
		}
	}

	return EC_SUCCESS;
}

/**************************************************************************************************/

<commit_msg>Added function for writing particles to EMP file. This feature is not being used be the commandline tool yet.<commit_after>/*
 * empsave.C
 *
 * .bgeo to .emp converter
 *
 * Copyright (c) 2010 Van Aarde Krynauw.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

#include "geo2emp.h"

#include <GEO/GEO_AttributeHandle.h>
#include <GEO/GEO_TriMesh.h>

#include <NgEmp.h>
#include <NgString.h>

/**************************************************************************************************/

Geo2Emp::ErrorCode Geo2Emp::saveMeshShape(Ng::Body*& pMeshBody)
{
	if (!_gdpOut)
	{
		//If we don't have a GDP for writing data into Houdini, return error.
		return EC_NULL_READ_GDP;
	}

	//Create a Naiad mesh body

	//Ng::Body* pMeshBody = new Ng::Body("trimesh");
	pMeshBody = new Ng::Body( _bodyName );
	pMeshBody->match("Mesh");

	//get the mutable shapes for the mesh body.
	Ng::TriangleShape& triShape( pMeshBody->mutableTriangleShape() );
	Ng::PointShape& ptShape( pMeshBody->mutablePointShape() );

	//For the sake of simplicity, copy ALL the points in the GDP across.
	//It will be much slower if we have to filter out all the non-mesh vertices and then remap the vertices to new point numbers.
	const GEO_PointList& ptlist = _gdpOut->points();
	const GEO_PrimList& primlist = _gdpOut->primitives();
	const GEO_Point* ppt;
	const GEO_Primitive* pprim;
	UT_Vector3 pos, v3;
	int numpoints = ptlist.entries();
	int vertcount = 0;
	GEO_AttributeHandle attr_v, attr_N;
	bool attr_v_valid, attr_N_valid;
	Ng::Buffer3f* pNormBuffer = 0;
	Ng::Buffer3f* pVelBuffer = 0;

	attr_v = _gdpOut->getPointAttribute("v");
	attr_N = _gdpOut->getPointAttribute("N");

	attr_v_valid = attr_v.isAttributeValid();
	attr_N_valid = attr_N.isAttributeValid();

	if (attr_v_valid)
	{
		//If the geometry has a velocity attribute, then create one on the mesh shape
		ptShape.createChannel3f( "velocity", em::vec3f(0.0f) );
		pVelBuffer = &(ptShape.mutableBuffer3f("velocity"));
		pVelBuffer->reserve(numpoints);
	}

	if (attr_N_valid)
	{
		//If the geometry has a normal attribute, then create one on the mesh shape
		ptShape.createChannel3f( "normal", em::vec3f(0.0f) );
		pNormBuffer = &(ptShape.mutableBuffer3f("normal"));
		pNormBuffer->reserve(numpoints);
	}

	//Retrieve the positions buffer from the point shape so that we can add all the vertices.
	Ng::Buffer3f& pPosBuf( ptShape.mutableBuffer3f("position") );
	pPosBuf.reserve( numpoints );

	for (int i = 0; i < numpoints; i++)
	{
		ppt = ptlist[i];
		pos = ppt->getPos();
		pPosBuf.push_back( em::vec3f( pos[0], pos[1], pos[2]) );

		//Set the velocity and normal attributes if there are any.
		if (attr_v_valid)
		{
			attr_v.setElement(ppt);
			v3 = attr_v.getV3();
			pVelBuffer->push_back( em::vec3f(v3[0], v3[1], v3[2]) );
		}

		if (attr_N_valid)
		{
			attr_N.setElement(ppt);
			v3 = attr_N.getV3();
			pNormBuffer->push_back( em::vec3f(v3[0], v3[1], v3[2]) );
		}

		//Ponder: Should we export blind data, or keep the attribute exports specific? Maybe blind exports from a SOP could work...
	}

	//Now its time to set the index channel to form faces
	Ng::Buffer3i& indexBuffer = triShape.mutableBuffer3i("index");
	em::vec3i triVec(0,0,0);

	for (pprim = primlist.head(); pprim; pprim = primlist.next(pprim) )
	{
		const GEO_TriMesh* triMesh = dynamic_cast<const GEO_TriMesh*>(pprim);

		if ( triMesh )
		{
			vertcount = triMesh->getVertexCount();

			if (vertcount >= 3)
			{
				//Export the first 3 vertices of this primitive.
				for (int i = 2; i >= 0; i--)
				{
					triVec[i] = triMesh->getVertex(i).getPt()->getNum();
				}
				indexBuffer.push_back( triVec );
			}
		}
	}

	return EC_SUCCESS;
}

/**************************************************************************************************/

Geo2Emp::ErrorCode Geo2Emp::saveParticleShape(Ng::Body*& pParticleBody)
{
	if (!_gdpOut)
	{
		//If we don't have a GDP for writing data into Houdini, return error.
		return EC_NULL_READ_GDP;
	}

	//Create a Naiad particle body
	//
	//Note from Marcus: the "quick and dirty" way of doing this is to just to make some small tile-layout,
	// add all particles into the first block, and then call "update" on the body so that it automatically
	// re-sorts them all into their appropriate blocks...
	pParticleBody = new Ng::Body( _bodyName );
	pParticleBody->match("Particle");

	//get the mutable shapes for the mesh body.
	Ng::ParticleShape& particleShape( pParticleBody->mutableParticleShape() );
	Ng::TileLayout& layout( pParticleBody->mutableLayout() );

	// PLEASE NOTE: this should really be a box close to a particle, instead of (0,0,0)...(1,1,1) but I was in a hurry!
	// so this will make some "dummy" tiles...
	layout.worldBoxRefine( em::vec3f(0,0,0), em::vec3f(1,1,1),1,false);
	particleShape.sync( layout );

	//For the sake of simplicity, copy ALL the points in the GDP across.
	//It will be much slower if we have to filter out all the non-mesh vertices and then remap the vertices to new point numbers.
	const GEO_PointList& ptlist = _gdpOut->points();
	const GEO_PrimList& primlist = _gdpOut->primitives();
	const GEO_Point* ppt;
	const GEO_Primitive* pprim;
	UT_Vector3 pos, v3;
	int numpoints = ptlist.entries();
	int vertcount = 0;
	GEO_AttributeHandle attr_v, attr_N;
	bool attr_v_valid, attr_N_valid;

	//Ng::Buffer3f* pNormBuffer = 0;
	//Ng::Buffer3f* pVelBuffer = 0;

	//The position attribute needs special/explicit treatment
	//Get the position buffer from the particleShape

	em::block3_array3f& vectorBlocks( particleShape.mutableBlocks3f("position") );
	em::block3vec3f& vectorData( vectorBlocks(0) );

	//Make sure we have enough space in the position attribute
	vectorData.reserve( _gdpOut->points().entries() );

	for (int i = 0; i < numpoints; i++)
	{
		//Set the point position
		pos = _gdpOut->points()[i]->getPos();
		vectorData.push_back( em::vec3f( pos[0], pos[1], pos[2] ) );
	}

	pParticleBody->update();

	return EC_SUCCESS;
}

/**************************************************************************************************/

<|endoftext|>


<commit_before> /*
 * esp32.cpp
 *
 *  Contains functions for ESP32 CPU's
 *
 *  This file is part of Arduino-IRremote https://github.com/z3t0/Arduino-IRremote.
 *
 *************************************************************************************
 * MIT License
 *
 * Copyright (c) 2020-2021 Rafi Khan, Armin Joachimsmeyer
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is furnished
 * to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
 * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
 * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
 * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 ************************************************************************************
 */
#ifdef ESP32

// This file contains functions specific to the ESP32.

#include "IRremote.h"

// "Idiot check"
#ifdef USE_DEFAULT_ENABLE_IR_IN
#error Must undef USE_DEFAULT_ENABLE_IR_IN
#endif

hw_timer_t *timer;
IRAM_ATTR void IRTimer(); // defined in IRremote.cpp, masqueraded as ISR(TIMER_INTR_NAME)

//+=============================================================================
// initialization
//
void IRrecv::enableIRIn() {
// Interrupt Service Routine - Fires every 50uS
    // ESP32 has a proper API to setup timers, no weird chip macros needed
    // simply call the readable API versions :)
    // 3 timers, choose #1, 80 divider nanosecond precision, 1 to count up
    timer = timerBegin(1, 80, 1);
    timerAttachInterrupt(timer, &IRTimer, 1);
    // every 50ns, autoreload = true
    timerAlarmWrite(timer, 50, true);
    timerAlarmEnable(timer);

    // Initialize state machine variables
    irparams.rcvstate = IR_REC_STATE_IDLE;
    irparams.rawlen = 0;

    // Set pin modes
    pinMode(irparams.recvpin, INPUT);
}

void IRrecv::disableIRIn() {
    timerEnd(timer);
    timerDetachInterrupt(timer);
}

void IRsend::enableIROut(int khz) {
    ledcSetup(LED_CHANNEL, khz * 1000, 8);  // 8 bit PWM resolution
    ledcAttachPin(IR_SEND_PIN, LED_CHANNEL); // bind pin to channel
}

#endif // ESP32
<commit_msg>Delete esp32.cpp<commit_after><|endoftext|>


<commit_before>/*
  Copyright (c) 2013 Matthew Stump

  This file is part of libprz.

  libprz is free software: you can redistribute it and/or modify
  it under the terms of the GNU Affero General Public License as
  published by the Free Software Foundation, either version 3 of the
  License, or (at your option) any later version.

  libprz is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU Affero General Public License for more details.

  You should have received a copy of the GNU Affero General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <boost/bind.hpp>
#include <stdint.h>
#include <boost/lexical_cast.hpp>

#include "encode.hpp"
#include "index_reader.hpp"
#include "index_writer.hpp"

#include "index.hpp"

// Required to use stdint.h
#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif

inline void
get_address(prz::index_address_t    position,
            prz::index_address_t*   bucket,
            prz::index_partition_t* bucket_index,
            prz::index_partition_t* bit_offset)
{
    *bucket = position >> 8; // div by 256
    prz::index_partition_t bucket_offset = position & 0xFF; // mod by 256
    *bucket_index = bucket_offset >> 6; // div by 64
    *bit_offset = bucket_offset & 0x3F; // mod by 64
}

inline void
set_bit(prz::index_segment_ptr input,
        prz::index_partition_t bucket_index,
        prz::index_partition_t bit_offset,
        bool                   val)
{
    input[bucket_index] |= (val ? 1 : 0) << bit_offset;
}

inline void
segment_union(prz::index_segment_ptr a,
              prz::index_segment_ptr b,
              prz::index_segment_ptr o)
{
    for (int i = 0; i < PRZ_INDEX_SEGMENT_LENGTH; ++i) {
        o[i] = a[i] | b[i];
    }
}

inline void
segment_intersection(prz::index_segment_ptr a,
                     prz::index_segment_ptr b,
                     prz::index_segment_ptr o)
{
    for (int i = 0; i < PRZ_INDEX_SEGMENT_LENGTH; ++i) {
        o[i] = a[i] & b[i];
    }
}

inline prz::index_t::iterator
find_insertion_point(prz::index_t::iterator begin,
                     prz::index_t::iterator end,
                     prz::index_address_t bucket)
{
    return std::find_if(begin,
                        end,
                        boost::bind(&prz::index_t::index_node_t::offset, _1 ) >= bucket);
}

inline prz::index_t::iterator
get_output_node(prz::index_t&          output,
                prz::index_t::iterator output_iter,
                prz::index_address_t        offset)
{
    output_iter = find_insertion_point(output_iter, output.end(), offset);
    if (output_iter == output.end()) {
        return output.insert(output_iter, new prz::index_t::index_node_t(offset));
    }
    return output_iter;
}

inline prz::status_t
union_behavior(prz::index_t& a_index,
               prz::index_t& b_index,
               prz::index_t& output)
{

    prz::index_t::iterator a_iter = a_index.begin();
    prz::index_t::iterator b_iter = b_index.begin();
    prz::index_t::iterator output_iter = output.begin();

    for (;;) {
        if (a_iter == a_index.end() && b_iter == b_index.end()) {
            break;
        }
        else if (a_iter == a_index.end()) {
            if (&output == &b_index) {
                break;
            }
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            memcpy(output_iter->segment, b_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++b_iter;
        }
        else if (a_iter->offset < b_iter->offset) {
            output_iter = get_output_node(output, output_iter, a_iter->offset);
            memcpy(output_iter->segment, a_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++a_iter;
        }
        else if (a_iter->offset > b_iter->offset) {
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            memcpy(output_iter->segment, b_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++b_iter;
        }
        else if (a_iter->offset == b_iter->offset) {
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            segment_union(a_iter->segment, b_iter->segment, output_iter->segment);
            ++a_iter;
            ++b_iter;
        }
        else {
            return prz::status_t(PRZ_ERROR_INDEX_OPERATION,
                                "shit's gone crazy in index union: "
                                + boost::lexical_cast<std::string>(a_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(b_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(output_iter->offset));
        }
    }
    return prz::status_t();
}

inline prz::status_t
intersection_behavior(prz::index_t& a_index,
                      prz::index_t& b_index,
                      prz::index_t& output)
{
    prz::index_t::iterator a_iter = a_index.begin();
    prz::index_t::iterator b_iter = b_index.begin();
    prz::index_t::iterator output_iter = output.begin();

    for (;;) {
        if (b_iter == b_index.end() || a_iter == a_index.end()) {
            output.erase(output_iter, output.end());
            break;
        }
        else if (a_iter->offset < b_iter->offset) {
            ++a_iter;
        }
        else if (a_iter->offset > b_iter->offset) {
            ++b_iter;
        }
        else if (a_iter->offset == b_iter->offset) {
            prz::index_t::iterator new_output_iter = get_output_node(output, output_iter, b_iter->offset);
            if (output_iter != output.end() && new_output_iter != output_iter) {
                output.erase(output_iter, new_output_iter);
            }
            output_iter = new_output_iter;
            segment_intersection(a_iter->segment, b_iter->segment, output_iter->segment);
            ++a_iter;
            ++b_iter;
            ++output_iter;
        }
        else {
            return prz::status_t(PRZ_ERROR_INDEX_OPERATION,
                                "shit's gone crazy in index intersection: "
                                + boost::lexical_cast<std::string>(a_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(b_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(output_iter->offset));
        }
    }
    return prz::status_t();
}

prz::index_t::index_node_t::index_node_t(const index_node_t& node) :
    offset(node.offset)
{
    memcpy(segment, node.segment, PRZ_INDEX_SEGMENT_SIZE);
}

prz::index_t::index_node_t::index_node_t(prz::index_address_t offset) :
    offset(offset)
{}

prz::index_t::index_node_t::index_node_t(prz::index_address_t offset,
                                         const index_segment_ptr data) :
    offset(offset)
{
    memcpy(segment, data, PRZ_INDEX_SEGMENT_SIZE);
}

void
prz::index_t::index_node_t::zero()
{
    memset(segment, 0, PRZ_INDEX_SEGMENT_SIZE);
}

prz::index_t::index_t(prz::index_partition_t partition,
                      const char*            field,
                      size_t                 field_size,
                      prz::index_address_t   value) :
    _partition(partition),
    _field(field, field + field_size),
    _value(value)
{}

prz::index_t::index_t(prz::index_partition_t partition,
                      const prz::byte_t*     field,
                      size_t                 field_size,
                      prz::index_address_t   value) :
    _partition(partition),
    _field(field, field + field_size),
    _value(value)
{}

prz::status_t
prz::index_t::execute(index_operation_enum operation,
                      prz::index_t&        a_index,
                      prz::index_t&        b_index,
                      prz::index_t&        output)
{
    if (operation == PRZ_INDEX_OP_INTERSECTION) {
        return intersection_behavior(a_index, b_index, output);
    }
    else if (operation == PRZ_INDEX_OP_UNION) {
        return union_behavior(a_index, b_index, output);
    }
    return prz::status_t(PRZ_ERROR_INDEX_OPERATION, "unkown/unsupported index operation");
}

prz::status_t
prz::index_t::execute(prz::index_operation_enum operation,
                      prz::index_reader_t*      reader,
                      prz::index_partition_t    partition,
                      const prz::byte_t*        field,
                      size_t                    field_size,
                      prz::index_address_t      value,
                      prz::index_t&             output)
{
    prz::status_t status = reader->read_index(partition, field, field_size, value, &output);
    if (status) {
        status = execute(operation, *this, output, output);
    }
    return status;
}

prz::status_t
prz::index_t::execute(prz::index_operation_enum operation,
                      prz::index_reader_t*      reader,
                      prz::index_partition_t    partition,
                      const prz::byte_t*        field,
                      size_t                    field_size,
                      prz::index_address_t      value)
{
    prz::index_t other_index(partition, field, field_size, value);
    prz::status_t status = reader->read_index(partition, field, field_size, value, &other_index);
    if (status) {
        status = execute(operation, *this, other_index, *this);
    }
    return status;
}

prz::status_t
prz::index_t::bit(prz::index_reader_t* reader,
                  prz::index_writer_t* writer,
                  prz::index_address_t bit,
                  bool                 state)
{
    prz::index_address_t   offset       = 0;
    prz::index_partition_t offset_index = 0;
    prz::index_partition_t bit_offset   = 0;
    get_address(bit, &offset, &offset_index, &bit_offset);

    prz::index_t::iterator it = find_insertion_point(begin(), end(), offset);

    prz::status_t status;
    if (it->offset != offset) {
        it = prz::index_t::iterator(_index.insert(it.base(), new index_node_t(offset)));
        status = reader->read_segment(_partition, &_field[0], _field.size(), _value, offset, it->segment);
    }

    if (status) {
        set_bit(it->segment, offset_index, bit_offset, state);
        status = writer->write_segment(_partition, &_field[0], _field.size(), _value, offset, it->segment);
    }
    return status;
}

bool
prz::index_t::bit(index_address_t      bit)
{
    prz::index_address_t        bucket       = 0;
    prz::index_partition_t      bucket_index = 0;
    prz::index_partition_t      bit_offset   = 0;
    get_address(bit, &bucket, &bucket_index, &bit_offset);

    prz::index_t::iterator it = find_insertion_point(begin(), end(), bucket);

    if (it->offset != bucket) {
        return false;
    }
    return (it->segment[bucket_index] & 1 << bit_offset);
}

prz::index_partition_t
prz::index_t::partition() const
{
    return _partition;
}

const prz::byte_t*
prz::index_t::field() const
{
    return &_field[0];
}

prz::index_address_t
prz::index_t::value() const
{
    return _value;
}
<commit_msg>removed unneeded #define from index.cpp<commit_after>/*
  Copyright (c) 2013 Matthew Stump

  This file is part of libprz.

  libprz is free software: you can redistribute it and/or modify
  it under the terms of the GNU Affero General Public License as
  published by the Free Software Foundation, either version 3 of the
  License, or (at your option) any later version.

  libprz is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU Affero General Public License for more details.

  You should have received a copy of the GNU Affero General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

#include <boost/bind.hpp>
#include <stdint.h>
#include <boost/lexical_cast.hpp>

#include "encode.hpp"
#include "index_reader.hpp"
#include "index_writer.hpp"

#include "index.hpp"

inline void
get_address(prz::index_address_t    position,
            prz::index_address_t*   bucket,
            prz::index_partition_t* bucket_index,
            prz::index_partition_t* bit_offset)
{
    *bucket = position >> 8; // div by 256
    prz::index_partition_t bucket_offset = position & 0xFF; // mod by 256
    *bucket_index = bucket_offset >> 6; // div by 64
    *bit_offset = bucket_offset & 0x3F; // mod by 64
}

inline void
set_bit(prz::index_segment_ptr input,
        prz::index_partition_t bucket_index,
        prz::index_partition_t bit_offset,
        bool                   val)
{
    input[bucket_index] |= (val ? 1 : 0) << bit_offset;
}

inline void
segment_union(prz::index_segment_ptr a,
              prz::index_segment_ptr b,
              prz::index_segment_ptr o)
{
    for (int i = 0; i < PRZ_INDEX_SEGMENT_LENGTH; ++i) {
        o[i] = a[i] | b[i];
    }
}

inline void
segment_intersection(prz::index_segment_ptr a,
                     prz::index_segment_ptr b,
                     prz::index_segment_ptr o)
{
    for (int i = 0; i < PRZ_INDEX_SEGMENT_LENGTH; ++i) {
        o[i] = a[i] & b[i];
    }
}

inline prz::index_t::iterator
find_insertion_point(prz::index_t::iterator begin,
                     prz::index_t::iterator end,
                     prz::index_address_t bucket)
{
    return std::find_if(begin,
                        end,
                        boost::bind(&prz::index_t::index_node_t::offset, _1 ) >= bucket);
}

inline prz::index_t::iterator
get_output_node(prz::index_t&          output,
                prz::index_t::iterator output_iter,
                prz::index_address_t        offset)
{
    output_iter = find_insertion_point(output_iter, output.end(), offset);
    if (output_iter == output.end()) {
        return output.insert(output_iter, new prz::index_t::index_node_t(offset));
    }
    return output_iter;
}

inline prz::status_t
union_behavior(prz::index_t& a_index,
               prz::index_t& b_index,
               prz::index_t& output)
{

    prz::index_t::iterator a_iter = a_index.begin();
    prz::index_t::iterator b_iter = b_index.begin();
    prz::index_t::iterator output_iter = output.begin();

    for (;;) {
        if (a_iter == a_index.end() && b_iter == b_index.end()) {
            break;
        }
        else if (a_iter == a_index.end()) {
            if (&output == &b_index) {
                break;
            }
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            memcpy(output_iter->segment, b_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++b_iter;
        }
        else if (a_iter->offset < b_iter->offset) {
            output_iter = get_output_node(output, output_iter, a_iter->offset);
            memcpy(output_iter->segment, a_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++a_iter;
        }
        else if (a_iter->offset > b_iter->offset) {
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            memcpy(output_iter->segment, b_iter->segment, PRZ_INDEX_SEGMENT_SIZE);
            ++b_iter;
        }
        else if (a_iter->offset == b_iter->offset) {
            output_iter = get_output_node(output, output_iter, b_iter->offset);
            segment_union(a_iter->segment, b_iter->segment, output_iter->segment);
            ++a_iter;
            ++b_iter;
        }
        else {
            return prz::status_t(PRZ_ERROR_INDEX_OPERATION,
                                "shit's gone crazy in index union: "
                                + boost::lexical_cast<std::string>(a_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(b_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(output_iter->offset));
        }
    }
    return prz::status_t();
}

inline prz::status_t
intersection_behavior(prz::index_t& a_index,
                      prz::index_t& b_index,
                      prz::index_t& output)
{
    prz::index_t::iterator a_iter = a_index.begin();
    prz::index_t::iterator b_iter = b_index.begin();
    prz::index_t::iterator output_iter = output.begin();

    for (;;) {
        if (b_iter == b_index.end() || a_iter == a_index.end()) {
            output.erase(output_iter, output.end());
            break;
        }
        else if (a_iter->offset < b_iter->offset) {
            ++a_iter;
        }
        else if (a_iter->offset > b_iter->offset) {
            ++b_iter;
        }
        else if (a_iter->offset == b_iter->offset) {
            prz::index_t::iterator new_output_iter = get_output_node(output, output_iter, b_iter->offset);
            if (output_iter != output.end() && new_output_iter != output_iter) {
                output.erase(output_iter, new_output_iter);
            }
            output_iter = new_output_iter;
            segment_intersection(a_iter->segment, b_iter->segment, output_iter->segment);
            ++a_iter;
            ++b_iter;
            ++output_iter;
        }
        else {
            return prz::status_t(PRZ_ERROR_INDEX_OPERATION,
                                "shit's gone crazy in index intersection: "
                                + boost::lexical_cast<std::string>(a_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(b_iter->offset)
                                + ":" + boost::lexical_cast<std::string>(output_iter->offset));
        }
    }
    return prz::status_t();
}

prz::index_t::index_node_t::index_node_t(const index_node_t& node) :
    offset(node.offset)
{
    memcpy(segment, node.segment, PRZ_INDEX_SEGMENT_SIZE);
}

prz::index_t::index_node_t::index_node_t(prz::index_address_t offset) :
    offset(offset)
{}

prz::index_t::index_node_t::index_node_t(prz::index_address_t offset,
                                         const index_segment_ptr data) :
    offset(offset)
{
    memcpy(segment, data, PRZ_INDEX_SEGMENT_SIZE);
}

void
prz::index_t::index_node_t::zero()
{
    memset(segment, 0, PRZ_INDEX_SEGMENT_SIZE);
}

prz::index_t::index_t(prz::index_partition_t partition,
                      const char*            field,
                      size_t                 field_size,
                      prz::index_address_t   value) :
    _partition(partition),
    _field(field, field + field_size),
    _value(value)
{}

prz::index_t::index_t(prz::index_partition_t partition,
                      const prz::byte_t*     field,
                      size_t                 field_size,
                      prz::index_address_t   value) :
    _partition(partition),
    _field(field, field + field_size),
    _value(value)
{}

prz::status_t
prz::index_t::execute(index_operation_enum operation,
                      prz::index_t&        a_index,
                      prz::index_t&        b_index,
                      prz::index_t&        output)
{
    if (operation == PRZ_INDEX_OP_INTERSECTION) {
        return intersection_behavior(a_index, b_index, output);
    }
    else if (operation == PRZ_INDEX_OP_UNION) {
        return union_behavior(a_index, b_index, output);
    }
    return prz::status_t(PRZ_ERROR_INDEX_OPERATION, "unkown/unsupported index operation");
}

prz::status_t
prz::index_t::execute(prz::index_operation_enum operation,
                      prz::index_reader_t*      reader,
                      prz::index_partition_t    partition,
                      const prz::byte_t*        field,
                      size_t                    field_size,
                      prz::index_address_t      value,
                      prz::index_t&             output)
{
    prz::status_t status = reader->read_index(partition, field, field_size, value, &output);
    if (status) {
        status = execute(operation, *this, output, output);
    }
    return status;
}

prz::status_t
prz::index_t::execute(prz::index_operation_enum operation,
                      prz::index_reader_t*      reader,
                      prz::index_partition_t    partition,
                      const prz::byte_t*        field,
                      size_t                    field_size,
                      prz::index_address_t      value)
{
    prz::index_t other_index(partition, field, field_size, value);
    prz::status_t status = reader->read_index(partition, field, field_size, value, &other_index);
    if (status) {
        status = execute(operation, *this, other_index, *this);
    }
    return status;
}

prz::status_t
prz::index_t::bit(prz::index_reader_t* reader,
                  prz::index_writer_t* writer,
                  prz::index_address_t bit,
                  bool                 state)
{
    prz::index_address_t   offset       = 0;
    prz::index_partition_t offset_index = 0;
    prz::index_partition_t bit_offset   = 0;
    get_address(bit, &offset, &offset_index, &bit_offset);

    prz::index_t::iterator it = find_insertion_point(begin(), end(), offset);

    prz::status_t status;
    if (it->offset != offset) {
        it = prz::index_t::iterator(_index.insert(it.base(), new index_node_t(offset)));
        status = reader->read_segment(_partition, &_field[0], _field.size(), _value, offset, it->segment);
    }

    if (status) {
        set_bit(it->segment, offset_index, bit_offset, state);
        status = writer->write_segment(_partition, &_field[0], _field.size(), _value, offset, it->segment);
    }
    return status;
}

bool
prz::index_t::bit(index_address_t      bit)
{
    prz::index_address_t        bucket       = 0;
    prz::index_partition_t      bucket_index = 0;
    prz::index_partition_t      bit_offset   = 0;
    get_address(bit, &bucket, &bucket_index, &bit_offset);

    prz::index_t::iterator it = find_insertion_point(begin(), end(), bucket);

    if (it->offset != bucket) {
        return false;
    }
    return (it->segment[bucket_index] & 1 << bit_offset);
}

prz::index_partition_t
prz::index_t::partition() const
{
    return _partition;
}

const prz::byte_t*
prz::index_t::field() const
{
    return &_field[0];
}

prz::index_address_t
prz::index_t::value() const
{
    return _value;
}
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <fstream>
#include <sstream>
#include <vector>
#include <regex>
#include <utility>
#include <ctype.h>
#include <chrono>
#include <cstring>
#include "lexer.h"
#include "tokens.h"

#define DEBUGLEX(a) if(debug_out){a}

Token EOFTOKEN {
	TokenType::eof, //type
	0, //col
	0, //line
	"EOF" //token
};

TokenType keyword_type[] = {
    TokenType::fn,
    TokenType::foreign,
    TokenType::import,
    TokenType::tchar,
    TokenType::tint,
    TokenType::tbool,
    TokenType::tfloat,
    TokenType::tdouble,
    TokenType::tvoid,
    TokenType::tuchar,
    TokenType::tchar,
    TokenType::tuint,
    TokenType::tint,
    TokenType::tfloat,
    TokenType::tdouble,
    TokenType::sif,
    TokenType::sfor,
    TokenType::swhile,
    TokenType::sdefer,
    TokenType::sreturn,
    TokenType::selse,
    TokenType::sbreak,
    TokenType::scontinue,
    TokenType::tstruct,
    TokenType::cast,
    TokenType::ssizeof
};

const char* keyword_array[] = {
    "fn",
    "extern",
    "import",
    "char",
    "int",
    "bool",
    "float",
    "double",
    "void",
    "u8",
    "s8",
    "u32",
    "s32",
    "f32",
    "f64",
    "if",
    "for",
    "while",
    "defer",
    "return",
    "else",
    "break",
    "continue",
    "struct",
    "cast",
    "sizeof"
};
unsigned int num_keywords = sizeof(keyword_array)/sizeof(const char*);

LexerTarget::LexerTarget(std::string name, bool debug) {
    filename = name;

    //auto start = std::chrono::steady_clock::now();
    //content = read_file(name);
    content= read_file(name);
    //auto finish = std::chrono::steady_clock::now();
    //auto diff = finish - start;
    //std::cout << "File Read Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(diff).count() << "ms\n";
    ln = content;

    lineNum = 0;
    colNum = 0;
    sub_begin = sub_len = 0;
    comment_depth = 0;
    f_idx = 0;
    debug_out = debug;
}

LexerTarget::~LexerTarget() {
}

bool LexerTarget::isDebug() {
    return debug_out;
}

std::string LexerTarget::targetName() {
    return filename;
}

void LexerTarget::lexcomment() {

    while(comment_depth > 0) {
        //std::cout << "Parsing comment block\n";

        if(content[f_idx] == '\0') {
            DEBUGLEX(std::cout << "Comment reaches the end of the file\n";)
            //we've reached end of file, stop
            comment_depth = 0;
            return;
        }

        while(content[f_idx] == 12 || content[f_idx] == 15) {
            //std::cout << ln << '\n';
            lineNum++;
            colNum = 0;
            f_idx++;
            ln = content+f_idx;
            if(content[f_idx] == '\0') {
                DEBUGLEX(std::cout << "Comment reaches the end of the file\n";)
                comment_depth = 0;
                return;
            }
        }

        if(content[f_idx] == '/') {
            if(content[f_idx+1] == '*') {
                ++comment_depth;
                colNum += 2;
                f_idx += 2;
                //std::cout << "Block comment begins. Depth " << comment_depth << "\n";
            }
        } else if(content[f_idx] == '*') {
            if(content[f_idx] == '/') {
                --comment_depth;
                colNum += 2;
                f_idx += 2;
                //std::cout << "Block comment ends. Depth " << comment_depth << "\n";
            }
        } else {
            ++colNum;
            ++f_idx;
        }
    }
}

Token LexerTarget::lex() {
    if(tokenizedFile.size()-2 > currentIdx) {
        Token ret = tokenizedFile[currentIdx+1];
        currentIdx++;
        return ret;
    }
    return tokenizedFile[currentIdx];
}

void LexerTarget::lexFile() {
    Token tok = lex_internal();
    //tok = lex_internal();
    tokenizedFile.reserve(200);
    while(tok.type != TokenType::eof) {
        //if(tokenizedFile.size() < 15) std::cout << tok.token << '\n';
        tokenizedFile.push_back(tok);
        tok = lex_internal();
    }
    tokenizedFile.push_back(tok);
    tokenizedFile.push_back(EOFTOKEN);
    currentIdx = 0;

    /*
    std::cout << "Lexer " << sizeof(LexerTarget) << '\n';
    std::cout << "Token " << sizeof(Token) << '\n';
    std::cout << "String " << sizeof(std::string) << '\n';
    std::cout << "const char* " << sizeof(const char*) << '\n';
    std::cout << "int " << sizeof(int) << '\n';
    std::cout << "TokenType " << sizeof(TokenType) << '\n';
    /**/
}

void printLn(char* s) {
    std::cout << "Ln is...\n";
    if(s[0] == '\0')
        return;

    while(s[0] != '\n') {
        std::cout << s[0];
        s++;
    }
    std::cout << "\n";
}

Token LexerTarget::lex_internal() {
    std::string token = "";

    TokenType longest_match_type;
    int longest_match = 0;

    /*
     * Match other tokens
     */

    char* remaining; // = ln+colNum;
    longest_match = 1; //default to 1
TOP:
    remaining = ln+colNum;
    switch(remaining[0]) {
        case ' ':
        case '\t':
            colNum++;
            f_idx++;
            goto TOP;
            break;
        case '\r':
        case '\n':
            lineNum++;
            colNum = 0;
            f_idx++;
            ln = content+f_idx;
            goto TOP;
            break;
        case '\0':
            return EOFTOKEN;
            break;
        case '(':
            longest_match_type = TokenType::lparen;
            break;
        case ')':
            longest_match_type = TokenType::rparen;
            break;
        case '.':
            longest_match_type = TokenType::dot;
            break;
        case ',':
            longest_match_type = TokenType::comma;
            break;
        case '{':
            longest_match_type = TokenType::lbrace;
            break;
        case '}':
            longest_match_type = TokenType::rbrace;
            break;
        case '[':
            longest_match_type = TokenType::lsqrbrace;
            break;
        case ']':
            longest_match_type = TokenType::rsqrbrace;
            break;
        case ':':
            {
                longest_match_type = TokenType::colon;
                if(remaining[1] == ':') {
                    longest_match++;
                    longest_match_type = TokenType::dblcolon;
                }
            }
            break;
        case ';':
            longest_match_type = TokenType::semicolon;
            break;
        case '@':
            longest_match_type = TokenType::dereference;
            break;
        case '~':
            longest_match_type = TokenType::tilda;
            break;
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
            {
                //TODO(marcus): throw lexer error if input is bad number is malformed
                int len_lit = 0;
                longest_match_type = TokenType::intlit;
                char current;
                do {
                    current = remaining[len_lit];
                    len_lit++;
                } while(current >= '0' && current <= '9');

                if(current == '.') {
                    //parse floating point
                    do {
                        current = remaining[len_lit];
                        len_lit++;
                    } while(current >= '0' && current <= '9');
                    len_lit--;
                    longest_match_type = TokenType::floatlit;
                    longest_match = len_lit;
                } else {
                    len_lit--;
                    longest_match = len_lit;
                }

            }
            break;
        case '"':
            {
                //TODO(marcus): handle errors/end of file
                int match_len = 1;
                bool matched = false;
                while(!matched) {
                    char c = content[f_idx+match_len];
                    if(c == 15 || c == 12) {
                        //handle multiline strings
                        lineNum++;
                        colNum = 0;
                    } if(c == '"' && content[f_idx+match_len-1] != '\\') {
                        matched = true;
                    }
                    match_len++;
                }
                longest_match = match_len;
                longest_match_type = TokenType::strlit;
            }
            break;
        case '\'':
            {
                int match_len = 1;
                bool matched = false;
                bool escaped = false;
                while(!matched) {
                    char c = content[f_idx+match_len];
                    if(c == 15 || c == 12) {
                        lineNum++;
                        colNum = 0;
                    }
                    if(escaped) {
                        switch(c) {
                            case 'n':
                            case 'a':
                            case '0':
                            case '\\':
                            case '\'':
                            case '\"':
                            case 'r':
                                break;
                            default:
                                std::cout << "Error, malformed character literal. Line " << lineNum << " Col " << colNum << '\n';
                                return EOFTOKEN;
                                break;
                        }
                    } else if(c == '\\') {
                        escaped = true;
                    } else if(c == '\'') {
                        matched = true;
                    }
                    match_len++;
                }
                longest_match = match_len;
                longest_match_type = TokenType::charlit;
            }
            break;
        case '+':
            {
                if(remaining[1] == '+') {
                    longest_match++;
                    longest_match_type = TokenType::increment;
                } else {
                    longest_match_type = TokenType::plus;
                }
            }
            break;
        case '-':
            {
                if(remaining[1] == '-') {
                    longest_match++;
                    longest_match_type = TokenType::increment;
                } else {
                    longest_match_type = TokenType::minus;
                }
            }
            break;
        case '/':
            {
                /*
                 * Check for block comments
                 */
                if(remaining[1] == '*') {
                    ++comment_depth;
                    colNum += 2;
                    f_idx += 2;
                    lexcomment();
                    return lex_internal();
                }

                /*
                 * Check for line comments
                 */
                else if(remaining[1] == '/') {
                    colNum += 2;
                    f_idx += 2;
                    while(remaining[colNum] != '\n' && remaining[colNum] != '\r') {
                        if(content[f_idx] == '\0') {
                            return EOFTOKEN;
                        }
                        colNum++;
                        f_idx++;
                    }
                    f_idx++;
                    colNum = 0;
                    lineNum++;
                    ln = content+f_idx;
                    //TODO(marcus): can probable replace this with goto
                    return lex_internal();
                }
                longest_match_type = TokenType::fslash;
            }
            break;
        case '*':
            {
                longest_match_type = TokenType::star;
            }
            break;
        case '^':
            {
                longest_match_type = TokenType::carrot;
            }
            break;
        case '%':
            {
                longest_match_type = TokenType::mod;
            }
            break;
        case '=':
            {
                longest_match_type = TokenType::assignment;
                if(remaining[1] == '=') {
                    longest_match = 2;
                    longest_match_type = TokenType::equality;
                }
            }
            break;
        case '<':
            {
                longest_match_type = TokenType::lessthan;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::ltequal;
                    longest_match = 2;
                }
            }
            break;
        case '>':
            {
                longest_match_type = TokenType::greaterthan;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::gtequal;
                    longest_match = 2;
                }
            }
            break;
        case '!':
            {
                longest_match_type = TokenType::exclaim;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::nequality;
                    longest_match = 2;
                }
            }
            break;
        case '|':
            {
                longest_match_type = TokenType::bar;
                if(remaining[1] == '|') {
                    longest_match_type = TokenType::dblbar;
                    longest_match = 2;
                }
            }
        case '&':
            {
                longest_match_type = TokenType::ampersand;
                if(remaining[1] == '&') {
                    longest_match_type = TokenType::dblampersand;
                    longest_match = 2;
                }
            }
            break;
        default:
            {
                //TODO(marcus): error checks
                int len_t = 0;
                char current=remaining[len_t];
                while(isalnum(current) || current == '_')
                {
                    len_t++;
                    current=remaining[len_t];
                }
                longest_match = len_t;
                longest_match_type = TokenType::id;
                std::string matched_string(ln+colNum,longest_match);
                for(unsigned int j = 0; j < num_keywords; j++) {
                   if(matched_string == keyword_array[j]) {
                       longest_match = strlen(keyword_array[j]);
                       longest_match_type = keyword_type[j];
                       break;
                   }
                }
            }
            break;
    }

    f_idx += longest_match;

    token = std::string(ln+colNum,longest_match);
    if(longest_match_type == TokenType::strlit) {
        std::regex stripped("\"([^\"]*)\"");
        std::smatch m;
        if(std::regex_match(token, m, stripped)) {
            token = m[1].str();
            size_t index;
            std::string newlines = "\\n";
            std::string newlinec = "\n";
            while( (index = token.find_first_of(newlines)) != std::string::npos) {
                token.replace(index, newlines.length(), newlinec);
                index += newlinec.length();
            }
        }
    }
    char* c_str_tok = (char*)malloc(token.size()+1);
    std::strcpy(c_str_tok,token.c_str());
    Token ret = {
		longest_match_type, //type
		colNum, //col
		lineNum, //line
		c_str_tok //token
	};
    colNum += longest_match;

    DEBUGLEX(std::cout << "token: " << token << "\n\n";)
    //std::cout << colNum << ' ' << f_idx << '\t' << content[f_idx] << '\n';
    return ret;

}

Token LexerTarget::peek() {
    return tokenizedFile[currentIdx];
}

Token LexerTarget::peekNext() {
    if(tokenizedFile.size()-2 > currentIdx) {
        return tokenizedFile[currentIdx+1];
    }
    return tokenizedFile[currentIdx];
}

char* read_file(const std::string& filename) {
    std::ifstream in(filename, std::ios::binary);
    in.seekg(0,std::ios::end);
    int size = in.tellg();
    char* content = (char*) malloc(size+1);
    in.seekg(0,std::ios::beg);
    in.read(content,size);
    content[size] = '\0';
    //std::cout << "File size was... " << size << '\n';
    return content;
}
<commit_msg>Fix bugs when parsing comments.<commit_after>#include <iostream>
#include <string>
#include <fstream>
#include <sstream>
#include <vector>
#include <regex>
#include <utility>
#include <ctype.h>
#include <chrono>
#include <cstring>
#include "lexer.h"
#include "tokens.h"

#define DEBUGLEX(a) if(debug_out){a}

Token EOFTOKEN {
	TokenType::eof, //type
	0, //col
	0, //line
	"EOF" //token
};

TokenType keyword_type[] = {
    TokenType::fn,
    TokenType::foreign,
    TokenType::import,
    TokenType::tchar,
    TokenType::tint,
    TokenType::tbool,
    TokenType::tfloat,
    TokenType::tdouble,
    TokenType::tvoid,
    TokenType::tuchar,
    TokenType::tchar,
    TokenType::tuint,
    TokenType::tint,
    TokenType::tfloat,
    TokenType::tdouble,
    TokenType::sif,
    TokenType::sfor,
    TokenType::swhile,
    TokenType::sdefer,
    TokenType::sreturn,
    TokenType::selse,
    TokenType::sbreak,
    TokenType::scontinue,
    TokenType::tstruct,
    TokenType::cast,
    TokenType::ssizeof
};

const char* keyword_array[] = {
    "fn",
    "extern",
    "import",
    "char",
    "int",
    "bool",
    "float",
    "double",
    "void",
    "u8",
    "s8",
    "u32",
    "s32",
    "f32",
    "f64",
    "if",
    "for",
    "while",
    "defer",
    "return",
    "else",
    "break",
    "continue",
    "struct",
    "cast",
    "sizeof"
};
unsigned int num_keywords = sizeof(keyword_array)/sizeof(const char*);

LexerTarget::LexerTarget(std::string name, bool debug) {
    filename = name;

    //auto start = std::chrono::steady_clock::now();
    //content = read_file(name);
    content= read_file(name);
    //auto finish = std::chrono::steady_clock::now();
    //auto diff = finish - start;
    //std::cout << "File Read Time: " << std::chrono::duration_cast<std::chrono::milliseconds>(diff).count() << "ms\n";
    ln = content;

    lineNum = 0;
    colNum = 0;
    sub_begin = sub_len = 0;
    comment_depth = 0;
    f_idx = 0;
    debug_out = debug;
}

LexerTarget::~LexerTarget() {
}

bool LexerTarget::isDebug() {
    return debug_out;
}

std::string LexerTarget::targetName() {
    return filename;
}

void LexerTarget::lexcomment() {

    while(comment_depth > 0) {
        //std::cout << "Parsing comment block\n";

        if(content[f_idx] == '\0') {
            DEBUGLEX(std::cout << "Comment reaches the end of the file\n";)
            //we've reached end of file, stop
            comment_depth = 0;
            return;
        }

        while(content[f_idx] == 12 || content[f_idx] == 15) {
            //std::cout << ln << '\n';
            lineNum++;
            colNum = 0;
            f_idx++;
            ln = content+f_idx;
            if(content[f_idx] == '\0') {
                DEBUGLEX(std::cout << "Comment reaches the end of the file\n";)
                comment_depth = 0;
                return;
            }
        }

        if(content[f_idx] == '/') {
            if(content[f_idx+1] == '*') {
                ++comment_depth;
                colNum += 2;
                f_idx += 2;
                //std::cout << "Block comment begins. Depth " << comment_depth << "\n";
            }
        } else if(content[f_idx] == '*') {
            if(content[f_idx+1] == '/') {
                --comment_depth;
                colNum += 2;
                f_idx += 2;
                //std::cout << "Block comment ends. Depth " << comment_depth << "\n";
            } else {
                ++colNum;
                ++f_idx;
            }
        } else {
            ++colNum;
            ++f_idx;
        }
    }
}

Token LexerTarget::lex() {
    if(tokenizedFile.size()-2 > currentIdx) {
        Token ret = tokenizedFile[currentIdx+1];
        currentIdx++;
        return ret;
    }
    return tokenizedFile[currentIdx];
}

void LexerTarget::lexFile() {
    Token tok = lex_internal();
    //tok = lex_internal();
    tokenizedFile.reserve(200);
    while(tok.type != TokenType::eof) {
        //if(tokenizedFile.size() < 15) std::cout << tok.token << '\n';
        tokenizedFile.push_back(tok);
        tok = lex_internal();
    }
    tokenizedFile.push_back(tok);
    tokenizedFile.push_back(EOFTOKEN);
    currentIdx = 0;

    /*
    std::cout << "Lexer " << sizeof(LexerTarget) << '\n';
    std::cout << "Token " << sizeof(Token) << '\n';
    std::cout << "String " << sizeof(std::string) << '\n';
    std::cout << "const char* " << sizeof(const char*) << '\n';
    std::cout << "int " << sizeof(int) << '\n';
    std::cout << "TokenType " << sizeof(TokenType) << '\n';
    /**/
}

void printLn(char* s) {
    std::cout << "Ln is...\n";
    if(s[0] == '\0')
        return;

    while(s[0] != '\n') {
        std::cout << s[0];
        s++;
    }
    std::cout << "\n";
}

Token LexerTarget::lex_internal() {
    std::string token = "";

    TokenType longest_match_type;
    int longest_match = 0;

    /*
     * Match other tokens
     */

    char* remaining; // = ln+colNum;
    longest_match = 1; //default to 1
TOP:
    remaining = ln+colNum;
    switch(remaining[0]) {
        case ' ':
        case '\t':
            colNum++;
            f_idx++;
            goto TOP;
            break;
        case '\r':
        case '\n':
            lineNum++;
            colNum = 0;
            f_idx++;
            ln = content+f_idx;
            goto TOP;
            break;
        case '\0':
            return EOFTOKEN;
            break;
        case '(':
            longest_match_type = TokenType::lparen;
            break;
        case ')':
            longest_match_type = TokenType::rparen;
            break;
        case '.':
            longest_match_type = TokenType::dot;
            break;
        case ',':
            longest_match_type = TokenType::comma;
            break;
        case '{':
            longest_match_type = TokenType::lbrace;
            break;
        case '}':
            longest_match_type = TokenType::rbrace;
            break;
        case '[':
            longest_match_type = TokenType::lsqrbrace;
            break;
        case ']':
            longest_match_type = TokenType::rsqrbrace;
            break;
        case ':':
            {
                longest_match_type = TokenType::colon;
                if(remaining[1] == ':') {
                    longest_match++;
                    longest_match_type = TokenType::dblcolon;
                }
            }
            break;
        case ';':
            longest_match_type = TokenType::semicolon;
            break;
        case '@':
            longest_match_type = TokenType::dereference;
            break;
        case '~':
            longest_match_type = TokenType::tilda;
            break;
        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':
            {
                //TODO(marcus): throw lexer error if input is bad number is malformed
                int len_lit = 0;
                longest_match_type = TokenType::intlit;
                char current;
                do {
                    current = remaining[len_lit];
                    len_lit++;
                } while(current >= '0' && current <= '9');

                if(current == '.') {
                    //parse floating point
                    do {
                        current = remaining[len_lit];
                        len_lit++;
                    } while(current >= '0' && current <= '9');
                    len_lit--;
                    longest_match_type = TokenType::floatlit;
                    longest_match = len_lit;
                } else {
                    len_lit--;
                    longest_match = len_lit;
                }

            }
            break;
        case '"':
            {
                //TODO(marcus): handle errors/end of file
                int match_len = 1;
                bool matched = false;
                while(!matched) {
                    char c = content[f_idx+match_len];
                    if(c == 15 || c == 12) {
                        //handle multiline strings
                        lineNum++;
                        colNum = 0;
                    } if(c == '"' && content[f_idx+match_len-1] != '\\') {
                        matched = true;
                    }
                    match_len++;
                }
                longest_match = match_len;
                longest_match_type = TokenType::strlit;
            }
            break;
        case '\'':
            {
                int match_len = 1;
                bool matched = false;
                bool escaped = false;
                while(!matched) {
                    char c = content[f_idx+match_len];
                    if(c == 15 || c == 12) {
                        lineNum++;
                        colNum = 0;
                    }
                    if(escaped) {
                        switch(c) {
                            case 'n':
                            case 'a':
                            case '0':
                            case '\\':
                            case '\'':
                            case '\"':
                            case 'r':
                                break;
                            default:
                                std::cout << "Error, malformed character literal. Line " << lineNum << " Col " << colNum << '\n';
                                return EOFTOKEN;
                                break;
                        }
                    } else if(c == '\\') {
                        escaped = true;
                    } else if(c == '\'') {
                        matched = true;
                    }
                    match_len++;
                }
                longest_match = match_len;
                longest_match_type = TokenType::charlit;
            }
            break;
        case '+':
            {
                if(remaining[1] == '+') {
                    longest_match++;
                    longest_match_type = TokenType::increment;
                } else {
                    longest_match_type = TokenType::plus;
                }
            }
            break;
        case '-':
            {
                if(remaining[1] == '-') {
                    longest_match++;
                    longest_match_type = TokenType::increment;
                } else {
                    longest_match_type = TokenType::minus;
                }
            }
            break;
        case '/':
            {
                /*
                 * Check for block comments
                 */
                if(remaining[1] == '*') {
                    ++comment_depth;
                    colNum += 2;
                    f_idx += 2;
                    lexcomment();
                    return lex_internal();
                }

                /*
                 * Check for line comments
                 */
                else if(remaining[1] == '/') {
                    int commentIdx = 2;
                    colNum += 2;
                    f_idx += 2;
                    while(remaining[commentIdx] != '\n' && remaining[commentIdx] != '\r') {
                        if(content[f_idx] == '\0') {
                            return EOFTOKEN;
                        }
                        colNum++;
                        f_idx++;
                        commentIdx++;
                    }
                    f_idx++;
                    colNum = 0;
                    lineNum++;
                    ln = content+f_idx;
                    //TODO(marcus): can probable replace this with goto
                    return lex_internal();
                }
                longest_match_type = TokenType::fslash;
            }
            break;
        case '*':
            {
                longest_match_type = TokenType::star;
            }
            break;
        case '^':
            {
                longest_match_type = TokenType::carrot;
            }
            break;
        case '%':
            {
                longest_match_type = TokenType::mod;
            }
            break;
        case '=':
            {
                longest_match_type = TokenType::assignment;
                if(remaining[1] == '=') {
                    longest_match = 2;
                    longest_match_type = TokenType::equality;
                }
            }
            break;
        case '<':
            {
                longest_match_type = TokenType::lessthan;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::ltequal;
                    longest_match = 2;
                }
            }
            break;
        case '>':
            {
                longest_match_type = TokenType::greaterthan;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::gtequal;
                    longest_match = 2;
                }
            }
            break;
        case '!':
            {
                longest_match_type = TokenType::exclaim;
                if(remaining[1] == '=') {
                    longest_match_type = TokenType::nequality;
                    longest_match = 2;
                }
            }
            break;
        case '|':
            {
                longest_match_type = TokenType::bar;
                if(remaining[1] == '|') {
                    longest_match_type = TokenType::dblbar;
                    longest_match = 2;
                }
            }
        case '&':
            {
                longest_match_type = TokenType::ampersand;
                if(remaining[1] == '&') {
                    longest_match_type = TokenType::dblampersand;
                    longest_match = 2;
                }
            }
            break;
        default:
            {
                //TODO(marcus): error checks
                int len_t = 0;
                char current=remaining[len_t];
                while(isalnum(current) || current == '_')
                {
                    len_t++;
                    current=remaining[len_t];
                }
                longest_match = len_t;
                longest_match_type = TokenType::id;
                std::string matched_string(ln+colNum,longest_match);
                for(unsigned int j = 0; j < num_keywords; j++) {
                   if(matched_string == keyword_array[j]) {
                       longest_match = strlen(keyword_array[j]);
                       longest_match_type = keyword_type[j];
                       break;
                   }
                }
            }
            break;
    }

    f_idx += longest_match;

    token = std::string(ln+colNum,longest_match);
    if(longest_match_type == TokenType::strlit) {
        std::regex stripped("\"([^\"]*)\"");
        std::smatch m;
        if(std::regex_match(token, m, stripped)) {
            token = m[1].str();
            size_t index;
            std::string newlines = "\\n";
            std::string newlinec = "\n";
            while( (index = token.find_first_of(newlines)) != std::string::npos) {
                token.replace(index, newlines.length(), newlinec);
                index += newlinec.length();
            }
        }
    }
    char* c_str_tok = (char*)malloc(token.size()+1);
    std::strcpy(c_str_tok,token.c_str());
    Token ret = {
		longest_match_type, //type
		colNum, //col
		lineNum, //line
		c_str_tok //token
	};
    colNum += longest_match;

    DEBUGLEX(std::cout << "token: " << token << "\n\n";)
    //std::cout << colNum << ' ' << f_idx << '\t' << content[f_idx] << '\n';
    return ret;

}

Token LexerTarget::peek() {
    return tokenizedFile[currentIdx];
}

Token LexerTarget::peekNext() {
    if(tokenizedFile.size()-2 > currentIdx) {
        return tokenizedFile[currentIdx+1];
    }
    return tokenizedFile[currentIdx];
}

char* read_file(const std::string& filename) {
    std::ifstream in(filename, std::ios::binary);
    in.seekg(0,std::ios::end);
    int size = in.tellg();
    char* content = (char*) malloc(size+1);
    in.seekg(0,std::ios::beg);
    in.read(content,size);
    content[size] = '\0';
    //std::cout << "File size was... " << size << '\n';
    return content;
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015-2018 Nagisa Sekiguchi
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "misc/fatal.h"
#include "misc/num.h"
#include "misc/hash.hpp"
#include "misc/flag_util.hpp"
#include "lexer.h"

namespace ydsh {

const char *toString(TokenKind kind) {
    const char *table[] = {
#define GEN_NAME(ENUM, STR) STR,
            EACH_TOKEN(GEN_NAME)
#undef GEN_NAME
    };
    return table[kind];
}

OperatorInfo getOpInfo(TokenKind kind) {
#define INFIX OperatorAttr::INFIX
#define PREFIX OperatorAttr::PREFIX
#define RASSOC OperatorAttr::RASSOC

    switch(kind) {
#define GEN_CASE(T, P, A) case T: return {P, A};
    EACH_OPERATOR(GEN_CASE)
#undef GEN_CASE
    default:
        return {};
    }

#undef INFIX
#undef PREFIX
#undef RASSOC
}

bool isAssignOp(TokenKind kind) {
    switch(kind) {
#define GEN_OP(K, P, A) case K: return true;
    EACH_ASSIGN_OPERATOR(GEN_OP)
#undef GEN_OP
    default:
        return false;
    }
}

const char *toModeName(LexerMode mode) {
    switch(mode) {
    case yycSTMT:
        return "STMT";
    case yycEXPR:
        return "EXPR";
    case yycNAME:
        return "NAME";
    case yycTYPE:
        return "TYPE";
    case yycCMD:
        return "CMD";
    case yycDSTRING:
        return "DSTRING";
    }
    return nullptr; // normally unreachable, but gcc requires this return statement.
}

// ###################
// ##     Lexer     ##
// ###################

bool Lexer::singleToString(Token token, std::string &out) const {
    if(this->startsWith(token, '$')) {
        return this->escapedSingleToString(token, out);
    }

    Token trimed = token;
    trimed.pos++;
    trimed.size -= 2;

    out = this->toTokenText(trimed);
    return true;
}

bool Lexer::escapedSingleToString(Token token, std::string &out) const {
    assert(this->withinRange(token));

    out.clear();
    out.reserve(token.size - 3);

    const unsigned int stopPos = token.pos + token.size - 1; // ignore suffix "'"
    for(unsigned int i = token.pos + 2; i < stopPos; i++) {  // ignore prefix "$'"
        char ch = this->buf[i];
        if(ch == '\\' && i + 1 < stopPos) {
            switch(this->buf[++i]) {
            case '\\':
                ch = '\\';
                break;
            case 'a':
                ch = '\a';
                break;
            case 'b':
                ch = '\b';
                break;
            case 'e':
            case 'E':
                ch = '\033';
                break;
            case 'f':
                ch = '\f';
                break;
            case 'n':
                ch = '\n';
                break;
            case 'r':
                ch = '\r';
                break;
            case 't':
                ch = '\t';
                break;
            case 'v':
                ch = '\v';
                break;
            case '\'':
                ch = '\'';
                break;
            case 'x':
                if(i + 1 < stopPos && isHex(this->buf[i + 1])) {
                    int v = hexToNum(this->buf[++i]);
                    if(i + 1 < stopPos && isHex(this->buf[i + 1])) {
                        v *= 16;
                        v += hexToNum(this->buf[++i]);
                        ch = static_cast<char>(v);
                        break;
                    }
                }
                return false;
            default:
                if(isOctal(this->buf[i])) {
                    int v = this->buf[i] - '0';
                    if(i + 1 < stopPos && isOctal(this->buf[i + 1])) {
                        v *= 8;
                        v += this->buf[++i] - '0';
                        if(i + 1 < stopPos && isOctal(this->buf[i + 1])) {
                            v *= 8;
                            v += this->buf[++i] - '0';
                            ch = static_cast<char>(v);
                            break;
                        }
                    }
                    return false;
                }
                --i;
                break;
            }
        }
        out += ch;
    }
    return true;
}

std::string Lexer::doubleElementToString(Token token) const {
    assert(this->withinRange(token));

    std::string str;
    str.reserve(token.size);

    const unsigned int stopPos = token.pos + token.size;
    for(unsigned int i = token.pos; i < stopPos; i++) {
        char ch = this->buf[i];
        if(ch == '\\' && i + 1 < stopPos) {
            char next = this->buf[++i];
            switch(next) {
            case '"':
            case '$':
            case '\\':
                ch = next;
                break;
            case '\n':
                continue;
            default:
                --i;
                break;
            }
        }
        str += ch;
    }
    return str;
}

std::string Lexer::toCmdArg(Token token) const {
    assert(this->withinRange(token));

    std::string str;
    str.reserve(token.size);

    for(unsigned int i = 0; i < token.size; i++) {
        char ch = this->buf[token.pos + i];
        if(ch == '\\') {
            char nextCh = this->buf[token.pos + ++i];
            switch(nextCh) {
            case '\n':
            case '\r':
                continue;
            default:
                ch = nextCh;
                break;
            }
        }
        str += ch;
    }
    return str;
}

std::string Lexer::toName(Token token) const {
    assert(this->withinRange(token));

    std::string name;
    name.reserve(token.size);
    for(unsigned int i = this->buf[token.pos] == '$' ? 1 : 0; i < token.size; i++) {
        char ch = this->buf[token.pos + i];
        switch(ch) {
        /**
         * ex. $true, ${true}, $@[
         */
        case '{':
        case '}':
        case '[':
            continue;
        default:
            name += ch;
            break;
        }
    }
    return name;
}

int Lexer::toInt32(Token token, int &status) const {
    if(this->isDecimal(token)) {
        long value = this->toInt64(token, status);
        if(value > INT32_MAX || value < INT32_MIN) {
            status = 1;
            return 0;
        }
        return static_cast<int>(value);
    }
    return static_cast<int>(this->toUint32(token, status));
}

unsigned int Lexer::toUint32(Token token, int &status) const {
    long value = this->toInt64(token, status);
    if(value > UINT32_MAX || value < 0) {
        status = 1;
        return 0;
    }
    return static_cast<unsigned int>(value);
}

long Lexer::toInt64(Token token, int &status) const {
    if(this->isDecimal(token)) {
        assert(this->withinRange(token));

        char str[token.size + 1];
        for(unsigned int i = 0; i < token.size; i++) {
            str[i] = this->buf[token.pos + i];
        }
        str[token.size] = '\0';

        long value = convertToInt64(str, status, 0, true);
        assert(status > -1);
        return value;
    }
    return static_cast<long>(this->toUint64(token, status));
}

unsigned long Lexer::toUint64(Token token, int &status) const {
    assert(this->withinRange(token));

    char str[token.size + 1];
    for(unsigned int i = 0; i < token.size; i++) {
        str[i] = this->buf[token.pos + i];
    }
    str[token.size] = '\0';

    unsigned long value = convertToUint64(str, status, 0, true);
    assert(status > -1);
    return value;
}

double Lexer::toDouble(Token token, int &status) const {
    assert(this->withinRange(token));

    char str[token.size + 1];
    for(unsigned int i = 0; i < token.size; i++) {
        str[i] = this->buf[token.pos + i];
    }
    str[token.size] = '\0';

    double value = convertToDouble(str, status);
    assert(status > -1);
    return value;
}

bool Lexer::isDecimal(Token token) const {
    assert(this->withinRange(token));
    if(token.size > 2) {    // '0x' or '0o'
        auto *str = reinterpret_cast<const char *>(this->buf.get() + token.pos);
        if(str[0] == '0' && (str[1] == 'x' || str[1] == 'o')) {
            return false;
        }
    }
    return true;
}

} // namespace ydsh<commit_msg>cleanup<commit_after>/*
 * Copyright (C) 2015-2018 Nagisa Sekiguchi
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "misc/fatal.h"
#include "misc/num.h"
#include "misc/hash.hpp"
#include "misc/flag_util.hpp"
#include "lexer.h"

namespace ydsh {

const char *toString(TokenKind kind) {
    const char *table[] = {
#define GEN_NAME(ENUM, STR) STR,
            EACH_TOKEN(GEN_NAME)
#undef GEN_NAME
    };
    return table[kind];
}

OperatorInfo getOpInfo(TokenKind kind) {
#define INFIX OperatorAttr::INFIX
#define PREFIX OperatorAttr::PREFIX
#define RASSOC OperatorAttr::RASSOC

    switch(kind) {
#define GEN_CASE(T, P, A) case T: return {P, A};
    EACH_OPERATOR(GEN_CASE)
#undef GEN_CASE
    default:
        return {};
    }

#undef INFIX
#undef PREFIX
#undef RASSOC
}

bool isAssignOp(TokenKind kind) {
    switch(kind) {
#define GEN_OP(K, P, A) case K: return true;
    EACH_ASSIGN_OPERATOR(GEN_OP)
#undef GEN_OP
    default:
        return false;
    }
}

const char *toModeName(LexerMode mode) {
    switch(mode) {
    case yycSTMT:
        return "STMT";
    case yycEXPR:
        return "EXPR";
    case yycNAME:
        return "NAME";
    case yycTYPE:
        return "TYPE";
    case yycCMD:
        return "CMD";
    case yycDSTRING:
        return "DSTRING";
    }
    return nullptr; // normally unreachable, but gcc requires this return statement.
}

// ###################
// ##     Lexer     ##
// ###################

bool Lexer::singleToString(Token token, std::string &out) const {
    if(this->startsWith(token, '$')) {
        return this->escapedSingleToString(token, out);
    }

    Token trimed = token;
    trimed.pos++;
    trimed.size -= 2;

    out = this->toTokenText(trimed);
    return true;
}

bool Lexer::escapedSingleToString(Token token, std::string &out) const {
    assert(this->withinRange(token));

    out.clear();
    out.reserve(token.size - 3);

    const unsigned int stopPos = token.pos + token.size - 1; // ignore suffix "'"
    for(unsigned int i = token.pos + 2; i < stopPos; i++) {  // ignore prefix "$'"
        char ch = this->buf[i];
        if(ch == '\\' && i + 1 < stopPos) {
            switch(this->buf[++i]) {
            case '\\':
                ch = '\\';
                break;
            case 'a':
                ch = '\a';
                break;
            case 'b':
                ch = '\b';
                break;
            case 'e':
            case 'E':
                ch = '\033';
                break;
            case 'f':
                ch = '\f';
                break;
            case 'n':
                ch = '\n';
                break;
            case 'r':
                ch = '\r';
                break;
            case 't':
                ch = '\t';
                break;
            case 'v':
                ch = '\v';
                break;
            case '\'':
                ch = '\'';
                break;
            case 'x':
                if(i + 1 < stopPos && isHex(this->buf[i + 1])) {
                    int v = hexToNum(this->buf[++i]);
                    if(i + 1 < stopPos && isHex(this->buf[i + 1])) {
                        v *= 16;
                        v += hexToNum(this->buf[++i]);
                        ch = static_cast<char>(v);
                        break;
                    }
                }
                return false;
            default:
                if(isOctal(this->buf[i])) {
                    int v = this->buf[i] - '0';
                    if(i + 1 < stopPos && isOctal(this->buf[i + 1])) {
                        v *= 8;
                        v += this->buf[++i] - '0';
                        if(i + 1 < stopPos && isOctal(this->buf[i + 1])) {
                            v *= 8;
                            v += this->buf[++i] - '0';
                            ch = static_cast<char>(v);
                            break;
                        }
                    }
                    return false;
                }
                --i;
                break;
            }
        }
        out += ch;
    }
    return true;
}

std::string Lexer::doubleElementToString(Token token) const {
    assert(this->withinRange(token));

    std::string str;
    str.reserve(token.size);

    const unsigned int stopPos = token.pos + token.size;
    for(unsigned int i = token.pos; i < stopPos; i++) {
        char ch = this->buf[i];
        if(ch == '\\' && i + 1 < stopPos) {
            char next = this->buf[++i];
            switch(next) {
            case '"':
            case '$':
            case '\\':
                ch = next;
                break;
            case '\n':
                continue;
            default:
                --i;
                break;
            }
        }
        str += ch;
    }
    return str;
}

std::string Lexer::toCmdArg(Token token) const {
    assert(this->withinRange(token));

    std::string str;
    str.reserve(token.size);

    for(unsigned int i = 0; i < token.size; i++) {
        char ch = this->buf[token.pos + i];
        if(ch == '\\') {
            char nextCh = this->buf[token.pos + ++i];
            switch(nextCh) {
            case '\n':
            case '\r':
                continue;
            default:
                ch = nextCh;
                break;
            }
        }
        str += ch;
    }
    return str;
}

std::string Lexer::toName(Token token) const {
    assert(this->withinRange(token));

    std::string name;
    name.reserve(token.size);
    for(unsigned int i = this->buf[token.pos] == '$' ? 1 : 0; i < token.size; i++) {
        char ch = this->buf[token.pos + i];
        switch(ch) {
        /**
         * ex. $true, ${true}, $@[
         */
        case '{':
        case '}':
        case '[':
            continue;
        default:
            name += ch;
            break;
        }
    }
    return name;
}

int Lexer::toInt32(Token token, int &status) const {
    if(this->isDecimal(token)) {
        long value = this->toInt64(token, status);
        if(value > INT32_MAX || value < INT32_MIN) {
            status = 1;
            return 0;
        }
        return static_cast<int>(value);
    }
    return static_cast<int>(this->toUint32(token, status));
}

unsigned int Lexer::toUint32(Token token, int &status) const {
    long value = this->toInt64(token, status);
    if(value > UINT32_MAX || value < 0) {
        status = 1;
        return 0;
    }
    return static_cast<unsigned int>(value);
}

long Lexer::toInt64(Token token, int &status) const {
    if(this->isDecimal(token)) {
        assert(this->withinRange(token));

        char str[token.size + 1];
        for(unsigned int i = 0; i < token.size; i++) {
            str[i] = this->buf[token.pos + i];
        }
        str[token.size] = '\0';

        long value = convertToInt64(str, status, 0, true);
        assert(status > -1);
        return value;
    }
    return static_cast<long>(this->toUint64(token, status));
}

unsigned long Lexer::toUint64(Token token, int &status) const {
    assert(this->withinRange(token));

    char str[token.size + 1];
    for(unsigned int i = 0; i < token.size; i++) {
        str[i] = this->buf[token.pos + i];
    }
    str[token.size] = '\0';

    unsigned long value = convertToUint64(str, status, 0, true);
    assert(status > -1);
    return value;
}

double Lexer::toDouble(Token token, int &status) const {
    assert(this->withinRange(token));
    double value = convertToDouble(this->toTokenText(token).c_str(), status);
    assert(status > -1);
    return value;
}

bool Lexer::isDecimal(Token token) const {
    assert(this->withinRange(token));
    if(token.size > 2) {    // '0x' or '0o'
        auto *str = reinterpret_cast<const char *>(this->buf.get() + token.pos);
        if(str[0] == '0' && (str[1] == 'x' || str[1] == 'o')) {
            return false;
        }
    }
    return true;
}

} // namespace ydsh<|endoftext|>


<commit_before>#pragma once

#define BTS_MAIL_INVENTORY_FETCH_LIMIT 4096
#define BTS_MAIL_MAX_MESSAGE_SIZE_BYTES (1024*1024)
#define BTS_MAIL_MAX_MESSAGE_AGE (fc::minutes(5))
#define BTS_MAIL_PROOF_OF_WORK_TARGET (fc::ripemd160("000000fffdeadbeeffffffffffffffffffffffff"))
#define BTS_MAIL_DEFAULT_MAIL_SERVERS (std::unordered_set<fc::ip::endpoint>({fc::ip::endpoint(fc::ip::address("107.170.30.182"), 1800)}))
<commit_msg>Set default mail server<commit_after>#pragma once

#define BTS_MAIL_INVENTORY_FETCH_LIMIT 4096
#define BTS_MAIL_MAX_MESSAGE_SIZE_BYTES (1024*1024)
#define BTS_MAIL_MAX_MESSAGE_AGE (fc::minutes(5))
#define BTS_MAIL_PROOF_OF_WORK_TARGET (fc::ripemd160("000ffffffdeadbeeffffffffffffffffffffffff"))
#define BTS_MAIL_DEFAULT_MAIL_SERVERS (std::unordered_set<fc::ip::endpoint>({fc::ip::endpoint(fc::ip::address("69.90.132.209"), 1376)}))
<|endoftext|>