zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-08-22 13:07:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

small cleanup

Browse Source
This commit is contained in:
dc-dc-dc 2023-12-17 21:44:54 -05:00
parent bd422decc4
commit be075a82bf
1 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
mlx/safetensor.cpp
View File

@ -18,23 +18,17 @@ Token Tokenizer::getToken() {
}
switch (nextChar) {
case '{':
this->_loc++;
return Token{TOKEN::CURLY_OPEN};
return Token{TOKEN::CURLY_OPEN, ++this->_loc};
case '}':
this->_loc++;
return Token{TOKEN::CURLY_CLOSE};
return Token{TOKEN::CURLY_CLOSE, ++this->_loc};
case ':':
this->_loc++;
return Token{TOKEN::COLON};
return Token{TOKEN::COLON, ++this->_loc};
case '[':
this->_loc++;
return Token{TOKEN::ARRAY_OPEN};
return Token{TOKEN::ARRAY_OPEN, ++this->_loc};
case ']':
this->_loc++;
return Token{TOKEN::ARRAY_CLOSE};
return Token{TOKEN::ARRAY_CLOSE, ++this->_loc};
case ',':
this->_loc++;
return Token{TOKEN::COMMA};
return Token{TOKEN::COMMA, ++this->_loc};
case '"': {
size_t start = ++this->_loc;
while (_data[++this->_loc] != '"' && this->hasMoreTokens())
@ -63,14 +57,16 @@ Token Tokenizer::getToken() {
JSONNode parseJson(const char* data, size_t len) {
auto tokenizer = Tokenizer(data, len);
std::stack<JSONNode*> ctx;
std::string key;
while (tokenizer.hasMoreTokens()) {
auto token = tokenizer.getToken();
switch (token.type) {
case TOKEN::NULL_TYPE:
break;
case TOKEN::CURLY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::OBJECT));
break;
case TOKEN::ARRAY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::LIST));
break;
case TOKEN::CURLY_CLOSE:
if (ctx.top()->is_type(JSONNode::Type::OBJECT)) {
auto obj = ctx.top();
@ -94,9 +90,7 @@ JSONNode parseJson(const char* data, size_t len) {
}
}
break;
case TOKEN::ARRAY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::LIST));
break;
case TOKEN::ARRAY_CLOSE:
if (ctx.top()->is_type(JSONNode::Type::STRING)) {
// key is above
@ -156,6 +150,8 @@ JSONNode parseJson(const char* data, size_t len) {
break;
case TOKEN::COLON:
break;
case TOKEN::NULL_TYPE:
break;
}
}
}