Switch to nlohmann's json lib

SConstruct

@@ -195,7 +195,7 @@ env = Environment(
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
     "#third_party/libyuv/include",
-    "#third_party/json11",
+    "#third_party/json",
     "#third_party/linux/include",
     "#third_party/snpe/include",
     "#third_party",
@@ -335,7 +335,7 @@ Export('env', 'qt_env', 'arch', 'real_arch')
 SConscript(['common/SConscript'])
 Import('_common', '_gpucommon')
 
-common = [_common, 'json11', 'zmq']
+common = [_common, 'json', 'zmq']
 gpucommon = [_gpucommon]
 
 Export('common', 'gpucommon')

common/SConscript

@@ -12,7 +12,7 @@ common_libs = [
 if arch != "Darwin":
   common_libs.append('gpio.cc')
 
-_common = env.Library('common', common_libs, LIBS="json11")
+_common = env.Library('common', common_libs, LIBS="json")
 
 files = [
   'clutil.cc',
@@ -24,10 +24,10 @@ Export('_common', '_gpucommon')
 if GetOption('extras'):
   env.Program('tests/test_common',
               ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
-              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+              LIBS=[_common, 'json', 'zmq', 'pthread'])
 
 # Cython bindings
-params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json'])
 
 SConscript([
   'transformations/SConscript',

common/logging_extra.py

@@ -5,18 +5,23 @@
 import json
 import time
 import uuid
+import locale
 import socket
 import logging
 import traceback
 from threading import local
 from collections import OrderedDict
 from contextlib import contextmanager
 
+# Set locale to use comma as decimal separator for testing
+# locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')
+
 LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
 
 def json_handler(obj):
-  # if isinstance(obj, (datetime.date, datetime.time)):
-  #   return obj.isoformat()
+  # Convert float numbers to use locale-specific formatting
+  if isinstance(obj, float):
+    return locale.str(obj)
   return repr(obj)
 
 def json_robust_dumps(obj):

common/swaglog.cc

@@ -11,7 +11,7 @@
 
 #include <zmq.h>
 #include <stdarg.h>
-#include "third_party/json11/json11.hpp"
+#include "third_party/json/json_helper.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
@@ -37,7 +37,7 @@ class SwaglogState {
       }
     }
 
-    ctx_j = json11::Json::object{};
+    ctx_j = json::Json::object{};
     if (char* dongle_id = getenv("DONGLE_ID")) {
       ctx_j["dongle_id"] = dongle_id;
     }
@@ -75,17 +75,17 @@ class SwaglogState {
   void* zctx = nullptr;
   void* sock = nullptr;
   int print_level;
-  json11::Json::object ctx_j;
+  json::Json::object ctx_j;
 };
 
 bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
 uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
 
 static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
-                            char* msg_buf, const json11::Json::object &msg_j={}) {
+                            char* msg_buf, const json::Json::object &msg_j={}) {
   static SwaglogState s;
 
-  json11::Json::object log_j = json11::Json::object {
+  json::Json::object log_j = json::Json::object {
     {"ctx", s.ctx_j},
     {"levelnum", levelnum},
     {"filename", filename},
@@ -101,7 +101,7 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons
 
   std::string log_s;
   log_s += (char)levelnum;
-  ((json11::Json)log_j).dump(log_s);
+  ((json::Json)log_j).dump(log_s);
   s.log(levelnum, filename, lineno, func, msg_buf, log_s);
 
   free(msg_buf);
@@ -124,14 +124,14 @@ void cloudlog_t_common(int levelnum, const char* filename, int lineno, const cha
   char* msg_buf = nullptr;
   int ret = vasprintf(&msg_buf, fmt, args);
   if (ret <= 0 || !msg_buf) return;
-  json11::Json::object tspt_j = json11::Json::object{
+  json::Json::object tspt_j = json::Json::object{
     {"event", msg_buf},
     {"time", std::to_string(nanos_since_boot())}
   };
   if (frame_id < NO_FRAME_ID) {
     tspt_j["frame_id"] = std::to_string(frame_id);
   }
-  tspt_j = json11::Json::object{{"timestamp", tspt_j}};
+  tspt_j = json::Json::object{{"timestamp", tspt_j}};
   cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
 }
 

common/tests/test_swaglog.cc

@@ -7,7 +7,7 @@
 #include "common/util.h"
 #include "common/version.h"
 #include "system/hardware/hw.h"
-#include "third_party/json11/json11.hpp"
+#include "third_party/json/json_helper.hpp"
 
 std::string daemon_name = "testy";
 std::string dongle_id = "test_dongle_id";
@@ -39,7 +39,7 @@ void recv_log(int thread_cnt, int thread_msg_cnt) {
 
     REQUIRE(buf[0] == CLOUDLOG_DEBUG);
     std::string err;
-    auto msg = json11::Json::parse(buf + 1, err);
+    auto msg = json::Json::parse(buf + 1, err);
     REQUIRE(!msg.is_null());
 
     REQUIRE(msg["levelnum"].int_value() == CLOUDLOG_DEBUG);

system/logcatd/logcatd_systemd.cc

@@ -5,7 +5,7 @@
 #include <map>
 #include <string>
 
-#include "third_party/json11/json11.hpp"
+#include "third_party/json/json.hpp"
 
 #include "cereal/messaging/messaging.h"
 #include "common/timing.h"
@@ -62,7 +62,7 @@ int main(int argc, char *argv[]) {
     // Build message
     auto androidEntry = msg.initEvent().initAndroidLog();
     androidEntry.setTs(timestamp);
-    androidEntry.setMessage(json11::Json(kv).dump());
+    androidEntry.setMessage(json::Json(kv).dump());
     if (kv.count("_PID")) androidEntry.setPid(std::atoi(kv["_PID"].c_str()));
     if (kv.count("PRIORITY")) androidEntry.setPriority(std::atoi(kv["PRIORITY"].c_str()));
     if (kv.count("SYSLOG_IDENTIFIER")) androidEntry.setTag(kv["SYSLOG_IDENTIFIER"]);

third_party/SConscript

@@ -1,4 +1,4 @@
 Import('env')
 
-env.Library('json11', ['json11/json11.cpp'], CCFLAGS=env['CCFLAGS'] + ['-Wno-unqualified-std-cast-call'])
+env.Library('json', ['json/json.cpp'], CCFLAGS=env['CCFLAGS'] + ['-Wno-unqualified-std-cast-call'])
 env.Library('kaitai', ['kaitai/kaitaistream.cpp'], CPPDEFINES=['KS_STR_ENCODING_NONE'])

third_party/json/json.cpp

@@ -0,0 +1,175 @@
+/* Copyright (c) 2013 Dropbox, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "json_helper.hpp"
+#include "json.hpp"
+
+namespace json {
+
+// Constructors
+Json::Json() noexcept : m_json(nullptr) {}
+Json::Json(std::nullptr_t) noexcept : m_json(nullptr) {}
+Json::Json(double value) : m_json(value) {}
+Json::Json(int value) : m_json(value) {}
+Json::Json(bool value) : m_json(value) {}
+Json::Json(const std::string &value) : m_json(value) {}
+Json::Json(std::string &&value) : m_json(std::move(value)) {}
+Json::Json(const char * value) : m_json(value) {}
+Json::Json(const array &values) : m_json(nlohmann::json::array()) {
+    for (const auto& v : values) {
+        m_json.push_back(v.m_json);
+    }
+}
+Json::Json(array &&values) : m_json(nlohmann::json::array()) {
+    for (auto& v : values) {
+        m_json.push_back(std::move(v.m_json));
+    }
+}
+Json::Json(const object &values) : m_json(nlohmann::json::object()) {
+    for (const auto& kv : values) {
+        m_json[kv.first] = kv.second.m_json;
+    }
+}
+Json::Json(object &&values) : m_json(nlohmann::json::object()) {
+    for (auto& kv : values) {
+        m_json[kv.first] = std::move(kv.second.m_json);
+    }
+}
+
+Json::Json(const nlohmann::json& j) : m_json(j) {}
+
+// Type accessors
+Json::Type Json::type() const {
+    if (m_json.is_null()) return NUL;
+    if (m_json.is_number()) return NUMBER;
+    if (m_json.is_boolean()) return BOOL;
+    if (m_json.is_string()) return STRING;
+    if (m_json.is_array()) return ARRAY;
+    if (m_json.is_object()) return OBJECT;
+    return NUL;
+}
+
+// Accessors
+double Json::number_value() const { return m_json.get<double>(); }
+int Json::int_value() const { return m_json.get<int>(); }
+bool Json::bool_value() const { return m_json.get<bool>(); }
+const std::string& Json::string_value() const {
+    static const std::string empty;
+    return m_json.is_string() ? m_json.get_ref<const std::string&>() : empty;
+}
+
+const Json::array& Json::array_items() const {
+    static const array empty;
+    if (!is_array()) return empty;
+    static thread_local array items;
+    items.clear();
+    for (const auto& item : m_json) {
+        items.emplace_back(item);
+    }
+    return items;
+}
+
+const Json::object& Json::object_items() const {
+    static const object empty;
+    if (!is_object()) return empty;
+    static thread_local object items;
+    items.clear();
+    for (auto it = m_json.begin(); it != m_json.end(); ++it) {
+        items.emplace(it.key(), Json(it.value()));
+    }
+    return items;
+}
+
+// Operators
+const Json& Json::operator[](size_t i) const {
+    static const Json empty;
+    if (!is_array() || i >= m_json.size()) return empty;
+    static thread_local Json result;
+    result = Json(m_json[i]);
+    return result;
+}
+
+const Json& Json::operator[](const std::string& key) const {
+    static const Json empty;
+    if (!is_object()) return empty;
+    auto it = m_json.find(key);
+    if (it == m_json.end()) return empty;
+    static thread_local Json result;
+    result = Json(*it);
+    return result;
+}
+
+bool Json::operator==(const Json& other) const {
+    return m_json == other.m_json;
+}
+
+bool Json::operator<(const Json& other) const {
+    return m_json < other.m_json;
+}
+
+// Parsing
+Json Json::parse(const std::string& in, std::string& err, JsonParse strategy) {
+    try {
+        auto result = nlohmann::json::parse(in);
+        err.clear();
+        return Json(result);
+    } catch (const nlohmann::json::parse_error& e) {
+        err = e.what();
+        return Json();
+    }
+}
+
+std::vector<Json> Json::parse_multi(const std::string& in, std::string::size_type& parser_stop_pos, std::string& err, JsonParse strategy) {
+    std::vector<Json> result;
+    try {
+        auto json_array = nlohmann::json::parse("[" + in + "]");
+        for (const auto& element : json_array) {
+            result.emplace_back(element);
+        }
+        parser_stop_pos = in.size();
+        err.clear();
+    } catch (const nlohmann::json::parse_error& e) {
+        err = e.what();
+    }
+    return result;
+}
+
+void Json::dump(std::string& out) const {
+    out = m_json.dump();
+}
+
+bool Json::has_shape(const shape& types, std::string& err) const {
+    if (!is_object()) {
+        err = "expected JSON object, got " + dump();
+        return false;
+    }
+
+    for (const auto& item : types) {
+        const auto it = m_json.find(item.first);
+        if (it == m_json.end() || Json(*it).type() != item.second) {
+            err = "bad type for " + item.first + " in " + dump();
+            return false;
+        }
+    }
+    return true;
+}
+
+} // namespace json