feature: add option to sort encoded object keys

Author: skewb1k

README.md

@@ -16,6 +16,7 @@ Table of Contents
     * [encode_number_precision](#encode_number_precision)
     * [encode_escape_forward_slash](#encode_escape_forward_slash)
     * [encode_skip_unsupported_value_types](#encode_skip_unsupported_value_types)
+    * [encode_sort_keys](#encode_sort_keys)
     * [decode_array_with_array_mt](#decode_array_with_array_mt)
 
 Description
@@ -201,6 +202,16 @@ This will generate:
 
 [Back to TOC](#table-of-contents)
 
+encode_sort_keys
+---------------------------
+**syntax:** `cjson.encode_sort_keys(enabled)`
+
+**default:** false
+
+If enabled, keys in encoded objects will be sorted in alphabetical order.
+
+[Back to TOC](#table-of-contents)
+
 decode_array_with_array_mt
 --------------------------
 **syntax:** `cjson.decode_array_with_array_mt(enabled)`

lua_cjson.c

@@ -91,6 +91,7 @@
 #define DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT 0
 #define DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH 1
 #define DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES 0
+#define DEFAULT_ENCODE_SORT_KEYS 0
 
 #ifdef DISABLE_INVALID_NUMBERS
 #undef DEFAULT_DECODE_INVALID_NUMBERS
@@ -155,6 +156,32 @@ static const char *json_token_type_name[] = {
     NULL
 };
 
+typedef struct {
+    strbuf_t *buf;
+    size_t offset;
+    size_t length;
+    int raw_typ;
+    union {
+        lua_Number number;
+        const char *string;
+    } raw;
+} key_entry_t;
+
+/* Stores all keys for a table when key sorting is enabled.
+ * - buf: buffer holding serialized key strings
+ * - keys: array of key_entry_t pointing into buf
+ * - size: number of keys stored
+ * - capacity: allocated capacity of keys array
+ */
+typedef struct {
+    strbuf_t buf;
+    key_entry_t *keys;
+    size_t size;
+    size_t capacity;
+} keybuf_t;
+
+#define KEYBUF_DEFAULT_CAPACITY 32
+
 typedef struct {
     json_token_type_t ch2token[256];
     char escape2char[256];  /* Decoding */
@@ -163,6 +190,10 @@ typedef struct {
      * encode_keep_buffer is set */
     strbuf_t encode_buf;
 
+    /* encode_keybuf is only allocated and used when
+     * sort_keys is set */
+    keybuf_t encode_keybuf;
+
     int encode_sparse_convert;
     int encode_sparse_ratio;
     int encode_sparse_safe;
@@ -172,6 +203,7 @@ typedef struct {
     int encode_keep_buffer;
     int encode_empty_table_as_object;
     int encode_escape_forward_slash;
+    int encode_sort_keys;
 
     int decode_invalid_numbers;
     int decode_max_depth;
@@ -449,6 +481,15 @@ static int json_cfg_encode_escape_forward_slash(lua_State *l)
     return ret;
 }
 
+static int json_cfg_encode_sort_keys(lua_State *l)
+{
+    json_config_t *cfg = json_arg_init(l, 1);
+
+    json_enum_option(l, 1, &cfg->encode_sort_keys, NULL, 1);
+
+    return 1;
+}
+
 static int json_destroy_config(lua_State *l)
 {
     json_config_t *cfg;
@@ -491,6 +532,7 @@ static void json_create_config(lua_State *l)
     cfg->decode_array_with_array_mt = DEFAULT_DECODE_ARRAY_WITH_ARRAY_MT;
     cfg->encode_escape_forward_slash = DEFAULT_ENCODE_ESCAPE_FORWARD_SLASH;
     cfg->encode_skip_unsupported_value_types = DEFAULT_ENCODE_SKIP_UNSUPPORTED_VALUE_TYPES;
+    cfg->encode_sort_keys = DEFAULT_ENCODE_SORT_KEYS;
 
 #if DEFAULT_ENCODE_KEEP_BUFFER > 0
     strbuf_init(&cfg->encode_buf, 0);
@@ -549,17 +591,17 @@ static void json_encode_exception(lua_State *l, json_config_t *cfg, strbuf_t *js
 {
     if (!cfg->encode_keep_buffer)
         strbuf_free(json);
+
+    if (cfg->encode_sort_keys) {
+        strbuf_free(&cfg->encode_keybuf.buf);
+        free(cfg->encode_keybuf.keys);
+    }
+
     luaL_error(l, "Cannot serialise %s: %s",
                   lua_typename(l, lua_type(l, lindex)), reason);
 }
 
-/* json_append_string args:
- * - lua_State
- * - JSON strbuf
- * - String (Lua stack index)
- *
- * Returns nothing. Doesn't remove string from Lua stack */
-static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
+static void json_append_string_contents(lua_State *l, strbuf_t *json, int lindex)
 {
     const char *escstr;
     const char *str;
@@ -572,19 +614,30 @@ static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
      * This buffer is reused constantly for small strings
      * If there are any excess pages, they won't be hit anyway.
      * This gains ~5% speedup. */
-    if (len > SIZE_MAX / 6 - 3)
+    if (len >= SIZE_MAX / 6)
         abort(); /* Overflow check */
-    strbuf_ensure_empty_length(json, len * 6 + 2);
+    strbuf_ensure_empty_length(json, len * 6);
 
-    strbuf_append_char_unsafe(json, '\"');
     for (i = 0; i < len; i++) {
         escstr = char2escape[(unsigned char)str[i]];
         if (escstr)
             strbuf_append_string(json, escstr);
         else
             strbuf_append_char_unsafe(json, str[i]);
     }
-    strbuf_append_char_unsafe(json, '\"');
+}
+
+/* json_append_string args:
+ * - lua_State
+ * - JSON strbuf
+ * - String (Lua stack index)
+ *
+ * Returns nothing. Doesn't remove string from Lua stack */
+static void json_append_string(lua_State *l, strbuf_t *json, int lindex)
+{
+    strbuf_append_char(json, '\"');
+    json_append_string_contents(l, json, lindex);
+    strbuf_append_char(json, '\"');
 }
 
 /* Find the size of the array on the top of the Lua stack
@@ -748,6 +801,15 @@ static void json_append_number(lua_State *l, json_config_t *cfg,
     strbuf_extend_length(json, len);
 }
 
+/* Compare key_entry_t for qsort. */
+static int cmp_key_entries(const void *a, const void *b) {
+    const key_entry_t *ka = (const key_entry_t *)a;
+    const key_entry_t *kb = (const key_entry_t *)b;
+    return memcmp(ka->buf->buf + ka->offset,
+                  kb->buf->buf + kb->offset,
+                  (ka->length < kb->length ? ka->length: kb->length));
+}
+
 static void json_append_object(lua_State *l, json_config_t *cfg,
                                int current_depth, strbuf_t *json)
 {
@@ -756,40 +818,118 @@ static void json_append_object(lua_State *l, json_config_t *cfg,
     /* Object */
     strbuf_append_char(json, '{');
 
-    lua_pushnil(l);
-    /* table, startkey */
     comma = 0;
-    while (lua_next(l, -2) != 0) {
-        json_pos = strbuf_length(json);
-        if (comma++ > 0)
-            strbuf_append_char(json, ',');
+    if (cfg->encode_sort_keys) {
+        keybuf_t *keybuf = &cfg->encode_keybuf;
+        size_t init_keybuf_size = cfg->encode_keybuf.size;
+        size_t init_keybuf_length = strbuf_length(&cfg->encode_keybuf.buf);
 
-        /* table, key, value */
-        keytype = lua_type(l, -2);
-        if (keytype == LUA_TNUMBER) {
-            strbuf_append_char(json, '"');
-            json_append_number(l, cfg, json, -2);
-            strbuf_append_mem(json, "\":", 2);
-        } else if (keytype == LUA_TSTRING) {
-            json_append_string(l, json, -2);
-            strbuf_append_char(json, ':');
-        } else {
-            json_encode_exception(l, cfg, json, -2,
-                                  "table key must be a number or string");
-            /* never returns */
+        lua_pushnil(l);
+        while (lua_next(l, -2) != 0) {
+            if (keybuf->size == keybuf->capacity){
+                if (!keybuf->capacity) {
+                    keybuf->capacity = KEYBUF_DEFAULT_CAPACITY;
+                    keybuf->keys = malloc(keybuf->capacity * sizeof(key_entry_t));
+                    if (!keybuf->keys)
+                        json_encode_exception(l, cfg, json, -1, "out of memory");
+                } else {
+                    keybuf->capacity *= 2;
+                    key_entry_t *tmp = realloc(keybuf->keys,
+                                               keybuf->capacity * sizeof(key_entry_t));
+                    if (!tmp)
+                        json_encode_exception(l, cfg, json, -1, "out of memory");
+                    keybuf->keys = tmp;
+                }
+            }
+
+            keytype = lua_type(l, -2);
+            key_entry_t key_entry = {
+                .buf = &keybuf->buf,
+                .offset = strbuf_length(&keybuf->buf),
+                .raw_typ = keytype,
+            };
+            if (keytype == LUA_TSTRING) {
+                json_append_string_contents(l, &keybuf->buf, -2);
+                key_entry.raw.string = lua_tostring(l, -2);
+            } else if (keytype == LUA_TNUMBER) {
+                json_append_number(l, cfg, &keybuf->buf, -2);
+                key_entry.raw.number = lua_tointeger(l, -2);
+            } else {
+                json_encode_exception(l, cfg, json, -2,
+                                      "table key must be number or string");
+            }
+            key_entry.length = strbuf_length(&keybuf->buf) - key_entry.offset;
+            keybuf->keys[keybuf->size++] = key_entry;
+            lua_pop(l, 1);
         }
 
-        /* table, key, value */
-        err = json_append_data(l, cfg, current_depth, json);
-        if (err) {
-            strbuf_set_length(json, json_pos);
-            if (comma == 1) {
-                comma = 0;
+        size_t keys_count = keybuf->size - init_keybuf_size;
+        qsort(keybuf->keys + init_keybuf_size, keys_count,
+              sizeof (key_entry_t), cmp_key_entries);
+
+        for (size_t i = init_keybuf_size; i < init_keybuf_size + keys_count; i++) {
+            key_entry_t *current_key = &keybuf->keys[i];
+            json_pos = strbuf_length(json);
+            if (comma++ > 0)
+                strbuf_append_char(json, ',');
+
+            strbuf_ensure_empty_length(json, current_key->length + 3);
+            strbuf_append_char_unsafe(json, '"');
+            strbuf_append_mem_unsafe(json, keybuf->buf.buf + current_key->offset,
+                                     current_key->length);
+            strbuf_append_mem(json, "\":", 2);
+
+            if (current_key->raw_typ == LUA_TSTRING)
+                lua_pushstring(l, current_key->raw.string);
+            else
+                lua_pushnumber(l, current_key->raw.number);
+
+            lua_gettable(l, -2);
+            err = json_append_data(l, cfg, current_depth, json);
+            if (err) {
+                strbuf_set_length(json, json_pos);
+                if (comma == 1)
+                    comma = 0;
             }
+            lua_pop(l, 1);
         }
+        /* resize encode_keybuf to reuse allocated memory for forward keys */
+        strbuf_set_length(&keybuf->buf, init_keybuf_length);
+        keybuf->size = init_keybuf_size;
+    } else {
+        lua_pushnil(l);
+        /* table, startkey */
+        while (lua_next(l, -2) != 0) {
+            json_pos = strbuf_length(json);
+            if (comma++ > 0)
+                strbuf_append_char(json, ',');
+
+            /* table, key, value */
+            keytype = lua_type(l, -2);
+            if (keytype == LUA_TNUMBER) {
+                strbuf_append_char(json, '"');
+                json_append_number(l, cfg, json, -2);
+                strbuf_append_mem(json, "\":", 2);
+            } else if (keytype == LUA_TSTRING) {
+                json_append_string(l, json, -2);
+                strbuf_append_char(json, ':');
+            } else {
+                json_encode_exception(l, cfg, json, -2,
+                                      "table key must be a number or string");
+                /* never returns */
+            }
 
-        lua_pop(l, 1);
-        /* table, key */
+            /* table, key, value */
+            err = json_append_data(l, cfg, current_depth, json);
+            if (err) {
+                strbuf_set_length(json, json_pos);
+                if (comma == 1)
+                    comma = 0;
+            }
+
+            lua_pop(l, 1);
+            /* table, key */
+        }
     }
 
     strbuf_append_char(json, '}');
@@ -914,6 +1054,12 @@ static int json_encode(lua_State *l)
         strbuf_reset(encode_buf);
     }
 
+    if (cfg->encode_sort_keys) {
+        strbuf_init(&cfg->encode_keybuf.buf, 0);
+        cfg->encode_keybuf.size = 0;
+        cfg->encode_keybuf.capacity = 0;
+    }
+
     json_append_data(l, cfg, 0, encode_buf);
     json = strbuf_string(encode_buf, &len);
 
@@ -922,6 +1068,11 @@ static int json_encode(lua_State *l)
     if (!cfg->encode_keep_buffer)
         strbuf_free(encode_buf);
 
+    if (cfg->encode_sort_keys) {
+        strbuf_free(&cfg->encode_keybuf.buf);
+        free(cfg->encode_keybuf.keys);
+    }
+
     return 1;
 }
 
@@ -1571,6 +1722,7 @@ static int lua_cjson_new(lua_State *l)
         { "decode_invalid_numbers", json_cfg_decode_invalid_numbers },
         { "encode_escape_forward_slash", json_cfg_encode_escape_forward_slash },
         { "encode_skip_unsupported_value_types", json_cfg_encode_skip_unsupported_value_types },
+        { "encode_sort_keys", json_cfg_encode_sort_keys },
         { "new", lua_cjson_new },
         { NULL, NULL }
     };

tests/test.lua

@@ -333,6 +333,34 @@ local cjson_tests = {
       json.decode, { [["\uDB00\uD"]] },
       false, { "Expected value but found invalid unicode escape code at character 2" } },
 
+    -- Test keys sorting
+    { "Set encode_sort_keys(true)",
+      json.encode_sort_keys, { true }, true, { true } },
+    { "Encode empty object with sorting",
+      json.encode, { {} },
+      true, { '{}' } },
+    { "Encode object with sorting",
+      json.encode, { { a = 0, b = 0, [1] = 0, ["$"] = 0, [4] = 0, ["%"] = 0 } },
+      true, { '{"$":0,"%":0,"1":0,"4":0,"a":0,"b":0}' } },
+    { "Encode object with string keys with sorting",
+      json.encode, { { aa = 1, ba = 3, ab = 2, bc = 4, cc = 5 } },
+      true, { '{"aa":1,"ab":2,"ba":3,"bc":4,"cc":5}' } },
+    { "Encode nested objects with sorting",
+      json.encode, { { a = { b = 2, a = 1, c = 3 }, c = 0, b = { b = { a = 0, b = 0 }, a = { a = 0, b = 0 } } } },
+      true, { '{"a":{"a":1,"b":2,"c":3},"b":{"a":{"a":0,"b":0},"b":{"a":0,"b":0}},"c":0}' } },
+    { "Encode array of objects with sorting",
+      json.encode, { {
+          { a = 0, [1] = 0, [4] = 0, b = 0 },
+          { f = 0, [5] = 0, [10] = 0, x = 0 },
+          { c = 0, [-2] = 0, [2] = 0, d = 0 },
+      } },
+      true, { '[{"1":0,"4":0,"a":0,"b":0},{"10":0,"5":0,"f":0,"x":0},{"-2":0,"2":0,"c":0,"d":0}]' } },
+    { "Encode object with unicode keys",
+      json.encode, { { ["é"] = 1, ["a"] = 2, ["ß"] = 3, ["中"] = 4 } },
+      true, { '{"a":2,"ß":3,"é":1,"中":4}' } },
+    { "Set encode_sort_keys(false)",
+      json.encode_sort_keys, { false }, true, { false } },
+
     -- Test locale support
     --
     -- The standard Lua interpreter is ANSI C online doesn't support locales