zhangyiss/mlx
1
0
Fork 0
mirror of https://github.com/ml-explore/mlx.git synced 2025-08-22 13:07:51 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

small cleanup

Browse Source
This commit is contained in:
dc-dc-dc 2023-12-17 21:44:54 -05:00
parent bd422decc4
commit be075a82bf
1 changed files with 13 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
mlx/safetensor.cpp
View File

@ -18,23 +18,17 @@ Token Tokenizer::getToken() {
} }
switch (nextChar) { switch (nextChar) {
case '{': case '{':
this->_loc++; return Token{TOKEN::CURLY_OPEN, ++this->_loc};
return Token{TOKEN::CURLY_OPEN};
case '}': case '}':
this->_loc++; return Token{TOKEN::CURLY_CLOSE, ++this->_loc};
return Token{TOKEN::CURLY_CLOSE};
case ':': case ':':
this->_loc++; return Token{TOKEN::COLON, ++this->_loc};
return Token{TOKEN::COLON};
case '[': case '[':
this->_loc++; return Token{TOKEN::ARRAY_OPEN, ++this->_loc};
return Token{TOKEN::ARRAY_OPEN};
case ']': case ']':
this->_loc++; return Token{TOKEN::ARRAY_CLOSE, ++this->_loc};
return Token{TOKEN::ARRAY_CLOSE};
case ',': case ',':
this->_loc++; return Token{TOKEN::COMMA, ++this->_loc};
return Token{TOKEN::COMMA};
case '"': { case '"': {
size_t start = ++this->_loc; size_t start = ++this->_loc;
while (_data[++this->_loc] != '"' && this->hasMoreTokens()) while (_data[++this->_loc] != '"' && this->hasMoreTokens())
@ -63,14 +57,16 @@ Token Tokenizer::getToken() {
JSONNode parseJson(const char* data, size_t len) { JSONNode parseJson(const char* data, size_t len) {
auto tokenizer = Tokenizer(data, len); auto tokenizer = Tokenizer(data, len);
std::stack<JSONNode*> ctx; std::stack<JSONNode*> ctx;
std::string key;
while (tokenizer.hasMoreTokens()) { while (tokenizer.hasMoreTokens()) {
auto token = tokenizer.getToken(); auto token = tokenizer.getToken();
switch (token.type) { switch (token.type) {
case TOKEN::NULL_TYPE:
break;
case TOKEN::CURLY_OPEN: case TOKEN::CURLY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::OBJECT)); ctx.push(new JSONNode(JSONNode::Type::OBJECT));
break; break;
case TOKEN::ARRAY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::LIST));
break;
case TOKEN::CURLY_CLOSE: case TOKEN::CURLY_CLOSE:
if (ctx.top()->is_type(JSONNode::Type::OBJECT)) { if (ctx.top()->is_type(JSONNode::Type::OBJECT)) {
auto obj = ctx.top(); auto obj = ctx.top();
@ -94,9 +90,7 @@ JSONNode parseJson(const char* data, size_t len) {
} }
} }
break; break;
case TOKEN::ARRAY_OPEN:
ctx.push(new JSONNode(JSONNode::Type::LIST));
break;
case TOKEN::ARRAY_CLOSE: case TOKEN::ARRAY_CLOSE:
if (ctx.top()->is_type(JSONNode::Type::STRING)) { if (ctx.top()->is_type(JSONNode::Type::STRING)) {
// key is above // key is above
@ -156,6 +150,8 @@ JSONNode parseJson(const char* data, size_t len) {
break; break;
case TOKEN::COLON: case TOKEN::COLON:
break; break;
case TOKEN::NULL_TYPE:
break;
} }
} }
} }