Serializing a table for filing I have a table in Lua, which contains two 1-dimensional arrays in which each array contains approximately 800,000 elements. I want to serialize this Lua table to file efficiently. Hence, I planned to use Lua C bindings.

#include "lua.h"
#include "lauxlib.h"
#include &lt;stdio.h&gt;
#include &lt;assert.h&gt;

static int do_it(lua_State *L) {
  assert(L &amp;&amp; lua_type(L, -1) == LUA_TTABLE);

  int len, idx;
  void *ptr;
  FILE *f;
  size_t r;

  lua_pushstring(L, "p");
  lua_gettable(L, -2);
  len = lua_objlen(L, -1);
  // instead of using lua_rawlen, i used lua_objlen. see below
  // len = lua_rawlen(L, -1); // it throws the following error
  // lua: error loading module 'savetable' from file './savetable.so':
  //    ./savetable.so: undefined symbol: lua_rawlen
  int p_values[len];
  for (idx = 1; idx &lt;= len; idx++) {
    lua_rawgeti(L, -1, idx);
    p_values[idx - 1] = (int)lua_tonumber(L, -1);
    lua_pop(L, 1);
  }

  f = fopen("p.bin", "wb");
  assert(f);
  r = fwrite(p_values, sizeof(int), len, f);
  printf("[p] wrote %zu elements out of %d requested\n", r, len);
  fclose(f);
  lua_pop(L, 1);

  lua_pushstring(L, "q");
  lua_gettable(L, -2);
  len = lua_objlen(L, -1);
  double q_values[len];
  for (idx = 1; idx &lt;= len; idx++) {
    lua_rawgeti(L, -1, idx);
    q_values[idx - 1] = (double)lua_tonumber(L, -1);
    lua_pop(L, 1);
  }

  f = fopen("q.bin", "wb");
  assert(f);
  r = fwrite(q_values, sizeof(double), len, f);
  printf("[q] wrote %zu elements out of %d requested\n", r, len);
  fclose(f);
  lua_pop(L, 1);

  return 1;
}

int luaopen_savetable(lua_State *L) {
  static const luaL_reg Map[] = {{"do_it", do_it}, {NULL, NULL}};

  luaL_register(L, "mytask", Map);
  return 1;
}


Please note that for debugging purpose, I have defined two very small 1-dimensional arrays:

my_table = {p = {11, 22, 33, 44}, q = {0.12, 0.23, 0.34, 0.45, 0.56}}

require "savetable"
mytask.do_it(my_table)


I used the following commands to compile and run it:

&gt; gcc -I/usr/include/lua5.1 -o savetable.so -shared savetable.c -fPIC
&gt; lua wrapper.lua


The code works, however, I am looking for suggestions to make table serialization to file much faster than current.

Please note that I am using Lua 5.1 on a 64-bit Ubuntu PC.
// undefined symbol: lua_rawlen


This happens because there is no such function in Lua 5.1 C API. As stated by documentation lua_objlen is preffered way to get table length.

FYI lua_rawlen function first appeared in Lua 5.2.

It is very handy to check Lua Reference manual to see availiable C API for particular Lua version.



Serialization with zero latency

Wow, that title stands out. Let me explain what I mean: serialization with zero latency means there is no serialization at all.

Watch out: it heavely depends on the origin of your table.

The caveats:


you must control the table creation
only inserts, no removes


If this is the case, follow on:

As you stated your table is actually an array of Lua numbers. In C Lua number is lua_Number which is probably a double value.

We can create a custom table implementation to only store Lua numbers in it.
The table will be continuos memory with lua_Number members in it.

typedef struct {
    lua_Number *items;
    size_t len; // number of entries in items
    size_t cap; // maximum `len` value before we will `realloc(items)`
} fasterarray;


To provide fasterarray we'll use lua_newuserdata. It will allocate memory that will be watched by Lua garbage collector. In particular when there is no more references the allocated userdata will be collected.

static int new(lua_State *L) {
    fasterarray *fa = lua_newuserdata(L, sizeof(fasterarray));
    luaL_getmetatable(L, NAME); // &lt;-- setting metatable, see below
    lua_setmetatable(L, -2);    // &lt;--
    fa-&gt;len = 0;
    fa-&gt;cap = 8;
    fa-&gt;items = calloc(fa-&gt;cap * sizeof(lua_Number), 1);
    return 1;
}


Insert function will realloc items when more space is needed.

static int insert(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    lua_Number num = lua_tonumber(L, 2);
    if (fa-&gt;len == fa-&gt;cap) {
        fa-&gt;cap *= 2;
        fa-&gt;items = realloc(fa-&gt;items, fa-&gt;cap * sizeof(lua_Number));
        assert(fa &amp;&amp; fa-&gt;items);
    }
    fa-&gt;items[fa-&gt;len++] = num;
    return 0;
}


Important part of this implementation is garbage collection. userdata will be collected by Lua automagically, but the memory behind items must be freed directly in C. To handle this we will use __gc metatable method.

static int gc(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    if (fa &amp;&amp; fa-&gt;items) free(fa-&gt;items);
    printf("gc\n");
    return 0;
}


Full example:

fasterarray.c

#include &lt;lua.h&gt;
#include &lt;lauxlib.h&gt;
#include &lt;stdlib.h&gt;
#include &lt;stdio.h&gt;
#include &lt;assert.h&gt;

#define NAME "fasterarray"

typedef struct {
    lua_Number *items;
    size_t len;
    size_t cap;
} fasterarray;

static int new(lua_State *L) {
    fasterarray *fa = lua_newuserdata(L, sizeof(fasterarray));
    luaL_getmetatable(L, NAME);
    lua_setmetatable(L, -2);
    fa-&gt;len = 0;
    fa-&gt;cap = 8;
    fa-&gt;items = calloc(fa-&gt;cap * sizeof(lua_Number), 1);
    return 1;
}

static int gc(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    if (fa &amp;&amp; fa-&gt;items) free(fa-&gt;items);
    printf("gc\n");
    return 0;
}

static int insert(lua_State *L) {
    fasterarray *fa = lua_touserdata(L, 1);
    lua_Number num = lua_tonumber(L, 2);
    if (fa-&gt;len == fa-&gt;cap) {
        fa-&gt;cap *= 2;
        fa-&gt;items = realloc(fa-&gt;items, fa-&gt;cap * sizeof(lua_Number));
        assert(fa &amp;&amp; fa-&gt;items);
    }
    fa-&gt;items[fa-&gt;len++] = num;
    return 0;
}

int luaopen_fasterarray(lua_State *L) {

    // fasterarray metatable
    luaL_newmetatable(L, NAME);
    luaL_register(L, NULL, (luaL_Reg []) {
        {"__gc", gc},
        {NULL, NULL}
    });

    // exported funcs
    luaL_register(L, NAME, (luaL_Reg []) {
        {"new", new},
        {"insert", insert},
        {NULL, NULL}
    });
    return 1;
}


example.lua

require"fasterarray"

local ar = fasterarray.new()
local i = 700000
while i &gt; 0 do
        fasterarray.insert(ar, i)
        i = i - 1
end