zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-08-21 20:46:46 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

parse the number from json

Browse Source
This commit is contained in:
dc-dc-dc 2023-12-18 09:53:50 -05:00
parent be075a82bf
commit d0285db98c
3 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
mlx/safetensor.cpp
View File

@ -36,7 +36,6 @@ Token Tokenizer::getToken() {
if (!this->hasMoreTokens()) { if (!this->hasMoreTokens()) {
throw new std::runtime_error("no more chars to parse"); throw new std::runtime_error("no more chars to parse");
} }
// pass the last "
return Token{TOKEN::STRING, start, ++this->_loc}; return Token{TOKEN::STRING, start, ++this->_loc};
} }
default: { default: {
@ -132,20 +131,24 @@ JSONNode parseJson(const char* data, size_t len) {
} }
break; break;
} }
case TOKEN::NUMBER: case TOKEN::NUMBER: {
// TODO: is there an easier way of doing this.
auto str = new std::string(data + token.start, token.end - token.start);
float val = strtof(str->c_str(), nullptr);
if (ctx.top()->is_type(JSONNode::Type::LIST)) { if (ctx.top()->is_type(JSONNode::Type::LIST)) {
ctx.top()->getList()->push_back(new JSONNode(JSONNode::Type::NUMBER)); ctx.top()->getList()->push_back(new JSONNode(val));
} else if (ctx.top()->is_type(JSONNode::Type::STRING)) { } else if (ctx.top()->is_type(JSONNode::Type::STRING)) {
auto key = ctx.top(); auto key = ctx.top();
ctx.pop(); ctx.pop();
if (ctx.top()->is_type(JSONNode::Type::OBJECT)) { if (ctx.top()->is_type(JSONNode::Type::OBJECT)) {
ctx.top()->getObject()->insert( ctx.top()->getObject()->insert(
{key->getString(), new JSONNode(JSONNode::Type::NUMBER)}); {key->getString(), new JSONNode(val)});
} else { } else {
throw new std::runtime_error("invalid json"); throw new std::runtime_error("invalid json");
} }
} }
break; break;
}
case TOKEN::COMMA: case TOKEN::COMMA:
break; break;
case TOKEN::COLON: case TOKEN::COLON:

12
mlx/safetensor.h
View File

@ -32,6 +32,9 @@ class JSONNode {
JSONNode(std::string* s) : _type(Type::STRING) { JSONNode(std::string* s) : _type(Type::STRING) {
this->_values.s = s; this->_values.s = s;
}; };
JSONNode(float f) : _type(Type::NUMBER) {
this->_values.f = f;
};
JSONObject* getObject() { JSONObject* getObject() {
if (!is_type(Type::OBJECT)) { if (!is_type(Type::OBJECT)) {
@ -54,6 +57,13 @@ class JSONNode {
return *this->_values.s; return *this->_values.s;
} }
float getNumber() {
if (!is_type(Type::NUMBER)) {
throw new std::runtime_error("not a number");
}
return this->_values.f;
}
inline bool is_type(Type t) { inline bool is_type(Type t) {
return this->_type == t; return this->_type == t;
} }
@ -67,7 +77,7 @@ class JSONNode {
JSONObject* object; JSONObject* object;
JSONList* list; JSONList* list;
std::string* s; std::string* s;
float fValue; float f;
} _values; } _values;
Type _type; Type _type;
}; };

7
tests/load_tests.cpp
View File

@ -38,13 +38,15 @@ TEST_CASE("test tokenizer") {
CHECK_EQ(tokenizer.getToken().type, io::TOKEN::NULL_TYPE); CHECK_EQ(tokenizer.getToken().type, io::TOKEN::NULL_TYPE);
} }
TEST_CASE("test load_safetensor") { TEST_CASE("test parseJson") {
auto raw = std::string("{}"); auto raw = std::string("{}");
auto res = io::parseJson(raw.c_str(), raw.size()); auto res = io::parseJson(raw.c_str(), raw.size());
CHECK(res.is_type(io::JSONNode::Type::OBJECT)); CHECK(res.is_type(io::JSONNode::Type::OBJECT));
raw = std::string("[]"); raw = std::string("[]");
res = io::parseJson(raw.c_str(), raw.size()); res = io::parseJson(raw.c_str(), raw.size());
CHECK(res.is_type(io::JSONNode::Type::LIST)); CHECK(res.is_type(io::JSONNode::Type::LIST));
raw = std::string("[{}, \"test\"]"); raw = std::string("[{}, \"test\"]");
res = io::parseJson(raw.c_str(), raw.size()); res = io::parseJson(raw.c_str(), raw.size());
CHECK(res.is_type(io::JSONNode::Type::LIST)); CHECK(res.is_type(io::JSONNode::Type::LIST));
@ -53,6 +55,7 @@ TEST_CASE("test load_safetensor") {
CHECK(res.getList()->at(1)->is_type(io::JSONNode::Type::STRING)); CHECK(res.getList()->at(1)->is_type(io::JSONNode::Type::STRING));
MESSAGE(res.getList()->at(1)->getString()); MESSAGE(res.getList()->at(1)->getString());
CHECK_EQ(res.getList()->at(1)->getString(), "test"); CHECK_EQ(res.getList()->at(1)->getString(), "test");
raw = std::string("{\"test\": \"test\", \"test_num\": 1}"); raw = std::string("{\"test\": \"test\", \"test_num\": 1}");
res = io::parseJson(raw.c_str(), raw.size()); res = io::parseJson(raw.c_str(), raw.size());
CHECK(res.is_type(io::JSONNode::Type::OBJECT)); CHECK(res.is_type(io::JSONNode::Type::OBJECT));
@ -60,6 +63,8 @@ TEST_CASE("test load_safetensor") {
CHECK(res.getObject()->at("test")->is_type(io::JSONNode::Type::STRING)); CHECK(res.getObject()->at("test")->is_type(io::JSONNode::Type::STRING));
CHECK_EQ(res.getObject()->at("test")->getString(), "test"); CHECK_EQ(res.getObject()->at("test")->getString(), "test");
CHECK(res.getObject()->at("test_num")->is_type(io::JSONNode::Type::NUMBER)); CHECK(res.getObject()->at("test_num")->is_type(io::JSONNode::Type::NUMBER));
CHECK_EQ(res.getObject()->at("test_num")->getNumber(), 1);
raw = std::string("{\"test\": { \"test\": \"test\"}}"); raw = std::string("{\"test\": { \"test\": \"test\"}}");
res = io::parseJson(raw.c_str(), raw.size()); res = io::parseJson(raw.c_str(), raw.size());
CHECK(res.is_type(io::JSONNode::Type::OBJECT)); CHECK(res.is_type(io::JSONNode::Type::OBJECT));