mirror of
https://github.com/antirez/gguf-tools.git
synced 2025-09-17 02:28:07 +08:00
ede59bb742
This change updates the data type definitions to be the same as the latest source code. Support for the bfloat16 data type is available however it can't interpret the IQ quantization formats yet. Cleanup of compiler warnings and other nits have been fixed, but behavioral changes have been avoided, and no new features are as of yet added.
568 lines
19 KiB
C
568 lines
19 KiB
C
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <ctype.h>
|
|
#include <string.h>
|
|
#include <assert.h>
|
|
#include <errno.h>
|
|
#include <math.h>
|
|
#include <inttypes.h>
|
|
|
|
#include "gguflib.h"
|
|
#include "sds.h"
|
|
#include "fp16.h"
|
|
|
|
/* Global options that can could be used for all the subcommands. */
|
|
struct {
|
|
int verbose; // --verbose option
|
|
} Opt = {0};
|
|
|
|
/* ========================== Utility functions ============================ */
|
|
|
|
/* Glob-style pattern matching. Return 1 on match, 0 otherwise. */
|
|
int strmatch(const char *pattern, int patternLen,
|
|
const char *string, int stringLen, int nocase)
|
|
{
|
|
while(patternLen && stringLen) {
|
|
switch(pattern[0]) {
|
|
case '*':
|
|
while (patternLen && pattern[1] == '*') {
|
|
pattern++;
|
|
patternLen--;
|
|
}
|
|
if (patternLen == 1)
|
|
return 1; /* match */
|
|
while(stringLen) {
|
|
if (strmatch(pattern+1, patternLen-1,
|
|
string, stringLen, nocase))
|
|
return 1; /* match */
|
|
string++;
|
|
stringLen--;
|
|
}
|
|
return 0; /* no match */
|
|
break;
|
|
case '?':
|
|
string++;
|
|
stringLen--;
|
|
break;
|
|
case '[':
|
|
{
|
|
int not, match;
|
|
|
|
pattern++;
|
|
patternLen--;
|
|
not = pattern[0] == '^';
|
|
if (not) {
|
|
pattern++;
|
|
patternLen--;
|
|
}
|
|
match = 0;
|
|
while(1) {
|
|
if (pattern[0] == '\\' && patternLen >= 2) {
|
|
pattern++;
|
|
patternLen--;
|
|
if (pattern[0] == string[0])
|
|
match = 1;
|
|
} else if (pattern[0] == ']') {
|
|
break;
|
|
} else if (patternLen == 0) {
|
|
pattern--;
|
|
patternLen++;
|
|
break;
|
|
} else if (patternLen >= 3 && pattern[1] == '-') {
|
|
int start = pattern[0];
|
|
int end = pattern[2];
|
|
int c = string[0];
|
|
if (start > end) {
|
|
int t = start;
|
|
start = end;
|
|
end = t;
|
|
}
|
|
if (nocase) {
|
|
start = tolower(start);
|
|
end = tolower(end);
|
|
c = tolower(c);
|
|
}
|
|
pattern += 2;
|
|
patternLen -= 2;
|
|
if (c >= start && c <= end)
|
|
match = 1;
|
|
} else {
|
|
if (!nocase) {
|
|
if (pattern[0] == string[0])
|
|
match = 1;
|
|
} else {
|
|
if (tolower((int)pattern[0]) == tolower((int)string[0]))
|
|
match = 1;
|
|
}
|
|
}
|
|
pattern++;
|
|
patternLen--;
|
|
}
|
|
if (not)
|
|
match = !match;
|
|
if (!match)
|
|
return 0; /* no match */
|
|
string++;
|
|
stringLen--;
|
|
break;
|
|
}
|
|
case '\\':
|
|
if (patternLen >= 2) {
|
|
pattern++;
|
|
patternLen--;
|
|
}
|
|
/* fall through */
|
|
default:
|
|
if (!nocase) {
|
|
if (pattern[0] != string[0])
|
|
return 0; /* no match */
|
|
} else {
|
|
if (tolower((int)pattern[0]) != tolower((int)string[0]))
|
|
return 0; /* no match */
|
|
}
|
|
string++;
|
|
stringLen--;
|
|
break;
|
|
}
|
|
pattern++;
|
|
patternLen--;
|
|
if (stringLen == 0) {
|
|
while(*pattern == '*') {
|
|
pattern++;
|
|
patternLen--;
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
if (patternLen == 0 && stringLen == 0)
|
|
return 1;
|
|
return 0;
|
|
}
|
|
|
|
/* ========================== 'show' subcommand ============================= */
|
|
|
|
void gguf_tools_show(const char *filename) {
|
|
gguf_ctx *ctx = gguf_open(filename);
|
|
if (ctx == NULL) {
|
|
perror(filename);
|
|
exit(1);
|
|
}
|
|
|
|
/* Show general information about the neural network. */
|
|
printf("%s (ver %d): %llu key-value pairs, %llu tensors\n",
|
|
filename,
|
|
(int)ctx->header->version,
|
|
(unsigned long long)ctx->header->metadata_kv_count,
|
|
(unsigned long long)ctx->header->tensor_count);
|
|
|
|
/* Show all the key-value pairs. */
|
|
gguf_key key;
|
|
while (gguf_get_key(ctx,&key)) {
|
|
printf("%.*s: [%s] ", (int)key.namelen, key.name, gguf_get_value_type_name(key.type));
|
|
gguf_print_value(ctx,key.type,key.val,Opt.verbose);
|
|
printf("\n");
|
|
}
|
|
|
|
/* Show all the tensors. */
|
|
gguf_tensor tensor;
|
|
uint64_t params = 0;
|
|
while (gguf_get_tensor(ctx,&tensor)) {
|
|
printf("%s tensor %.*s @%" PRIu64 ", %" PRIu64 " weights, dims ",
|
|
gguf_get_tensor_type_name(tensor.type),
|
|
(int)tensor.namelen,
|
|
tensor.name,
|
|
tensor.offset,
|
|
tensor.num_weights);
|
|
for (uint32_t j = 0; j < tensor.ndim; j++) {
|
|
printf("%s%" PRIu64 "",(j == 0) ? "[" : ",", tensor.dim[j]);
|
|
}
|
|
printf("], %" PRIu64 " bytes\n", tensor.bsize);
|
|
|
|
params += tensor.num_weights;
|
|
}
|
|
printf("gguf-tools.info.parameters: %.02fB\n",
|
|
(double)params/1000000000);
|
|
return;
|
|
}
|
|
|
|
/* ======================= 'split-mixtral' subcommand ======================= */
|
|
|
|
/* Read a Mixtral MoE model and creates a new non-MoE GGUF file based
|
|
* on the weights of the experts with IDs in the array of 'experts_id'.
|
|
* The array must contain 32 integers, one for each layer. */
|
|
void gguf_tools_split_mixtral(int *experts_id, const char *mixtral_filename, const char *output_filename) {
|
|
gguf_ctx *mixtral = gguf_open(mixtral_filename);
|
|
if (mixtral == NULL) {
|
|
perror(mixtral_filename);
|
|
exit(1);
|
|
}
|
|
|
|
gguf_ctx *output = gguf_create(output_filename, GGUF_NONE);
|
|
if (output == NULL) {
|
|
perror(output_filename);
|
|
exit(1);
|
|
}
|
|
|
|
/* To start, copy all the key value items, excluding the one
|
|
* related to the experts. */
|
|
gguf_key key;
|
|
while (gguf_get_key(mixtral,&key)) {
|
|
char keybuf[1024];
|
|
snprintf(keybuf,sizeof(keybuf),"%.*s",(int)key.namelen, key.name);
|
|
|
|
int skip = strstr(keybuf,"llama.expert_") != NULL;
|
|
|
|
if (!skip)
|
|
printf("Copying %s\n", keybuf);
|
|
uint64_t value_start_offset = mixtral->off;
|
|
void *value = mixtral->data+mixtral->off;
|
|
// Just consume the value without doing anything with it.
|
|
gguf_do_with_value(mixtral,key.type,key.val,NULL,0,0,NULL);
|
|
uint64_t value_len = mixtral->off - value_start_offset;
|
|
|
|
// Now append the value to the output model.
|
|
if (!skip)
|
|
gguf_append_kv(output,key.name,key.namelen,key.type,value,value_len);
|
|
}
|
|
|
|
/* Now it's time to copy the tensors. We need to copy all the shared
|
|
* tensors (between the different experts), but only a set of
|
|
* expert-specific tensors corresponding to the expert ID the user
|
|
* wants to extract. */
|
|
struct tensor_to_copy {
|
|
sds dest_name; // Tensor name in the output file.
|
|
gguf_tensor orig_info; // Original tensor info.
|
|
uint64_t dest_offset; // Destination offset in output file.
|
|
uint64_t size; // Tensor total bytes.
|
|
};
|
|
|
|
uint32_t num_tensors = 0;
|
|
uint32_t max_tensors = 2048;
|
|
|
|
struct tensor_to_copy *tensors =
|
|
malloc(sizeof(struct tensor_to_copy)*max_tensors);
|
|
if (tensors == NULL) {
|
|
perror("Allocating tensors info array");
|
|
exit(1);
|
|
}
|
|
|
|
/* Scan Mixtral tensors looking for the ones we need to copy
|
|
* in the output model. */
|
|
gguf_tensor tensor_info;
|
|
while (gguf_get_tensor(mixtral,&tensor_info)) {
|
|
assert(num_tensors < max_tensors);
|
|
|
|
char tn[1024]; // Tensor name as null terminated string.
|
|
snprintf(tn,sizeof(tn),"%.*s",(int)tensor_info.namelen, tensor_info.name);
|
|
|
|
/* The tensor is a feed-forward tensor? We want to copy only
|
|
* the ones of our expert ID. */
|
|
if (strstr(tn,".ffn_") != NULL && strstr(tn,".ffn_norm") == NULL) {
|
|
/* Extract which block this FFN belongs. */
|
|
int block;
|
|
assert(memcmp(tn,"blk.",4) == 0); // Must start with blk.<block>
|
|
char *p = strchr(tn+4,'.');
|
|
assert(p != NULL);
|
|
*p = 0;
|
|
block = atoi(tn+4);
|
|
*p = '.';
|
|
assert(block >= 0 && block < 32);
|
|
|
|
/* Now that we have the block, we can select the corresponding
|
|
* expert ID we want to use for this block. */
|
|
int expert_id = experts_id[block];
|
|
|
|
char match[32];
|
|
snprintf(match,sizeof(match),".%d.weight",expert_id);
|
|
char *match_ptr = strstr(tn,match);
|
|
if (match_ptr == NULL) {
|
|
printf("Skipping tensor %s\n", tn);
|
|
continue; // Skip this tensor.
|
|
}
|
|
|
|
/* We need to remove the .<id>. from the name. */
|
|
size_t taillen = strlen(match_ptr);
|
|
memmove(match_ptr,match_ptr+2,taillen+1);
|
|
}
|
|
|
|
/* Create the entry for this tensor. Later we will scan all our
|
|
* entries and append data to our output tensor. */
|
|
tensors[num_tensors].dest_name = sdsnew(tn);
|
|
if (tensors[num_tensors].dest_name == NULL) {
|
|
perror("Allocating test tensor name");
|
|
exit(1);
|
|
}
|
|
tensors[num_tensors].orig_info = tensor_info;
|
|
tensors[num_tensors].size = tensor_info.bsize;
|
|
num_tensors++;
|
|
}
|
|
|
|
/* Now we need to set the offset for our destination tensors. As
|
|
* we calculate the offsets, we can emit the tensors information
|
|
* section as well. */
|
|
uint64_t tensor_off = 0; // Tensor offsets are relative to data section,
|
|
// so we start at offset 0.
|
|
for (uint32_t j = 0; j < num_tensors; j++) {
|
|
/* Align offset. */
|
|
tensor_off += gguf_get_alignment_padding(mixtral->alignment,tensor_off);
|
|
tensors[j].dest_offset = tensor_off;
|
|
if (gguf_append_tensor_info(output,tensors[j].dest_name,strlen(tensors[j].dest_name),tensors[j].orig_info.ndim,tensors[j].orig_info.dim,tensors[j].orig_info.type,tensor_off) == 0)
|
|
{
|
|
perror("Failed to append tensor info");
|
|
exit(1);
|
|
}
|
|
tensor_off += tensors[j].orig_info.bsize;
|
|
}
|
|
printf("Output file: after writing tensors info, file size is: %" PRIu64 "\n", output->size);
|
|
|
|
/* Finally, append the tensors weights. */
|
|
for (uint32_t j = 0; j < num_tensors; j++) {
|
|
printf("Writing tensor %s (weights from %.*s)\n", tensors[j].dest_name,
|
|
(int)tensors[j].orig_info.namelen, tensors[j].orig_info.name);
|
|
if (gguf_append_tensor_data(output,tensors[j].orig_info.weights_data,
|
|
tensors[j].orig_info.bsize) == 0)
|
|
{
|
|
perror("Failed to append tensor data");
|
|
exit(1);
|
|
}
|
|
}
|
|
exit(0);
|
|
}
|
|
|
|
/* ====================== 'inspect-weights' subcommand ====================== */
|
|
|
|
void gguf_tools_inspect_weights(const char *filename, const char *tname, uint64_t count) {
|
|
gguf_ctx *ctx = gguf_open(filename);
|
|
if (ctx == NULL) {
|
|
perror(filename);
|
|
exit(1);
|
|
}
|
|
|
|
/* Skip all the key-value pairs. */
|
|
gguf_skip_key_values_section(ctx);
|
|
|
|
/* Look for the tensor with the specified name. */
|
|
size_t tnamelen = strlen(tname);
|
|
gguf_tensor tensor;
|
|
while (gguf_get_tensor(ctx,&tensor)) {
|
|
if (tensor.namelen != tnamelen ||
|
|
memcmp(tensor.name,tname,tnamelen)) continue;
|
|
break; // Matching tensor found!
|
|
}
|
|
|
|
if (tensor.name == NULL) {
|
|
fprintf(stderr, "A tensor with the specified name was not found\n");
|
|
exit(1);
|
|
}
|
|
|
|
float *weights = gguf_tensor_to_float(&tensor);
|
|
if (weights == NULL) {
|
|
if (errno == EINVAL) {
|
|
fprintf(stderr,"Unsupported tensor type: %s\n",
|
|
gguf_get_tensor_type_name(tensor.type));
|
|
} else {
|
|
fprintf(stderr,"Out of memory\n");
|
|
}
|
|
exit(1);
|
|
}
|
|
|
|
uint64_t strides[GGUF_TENSOR_MAX_DIM] = {0};
|
|
strides[tensor.ndim-1] = 1;
|
|
for (int j = tensor.ndim - 2; j >= 0; j--) {
|
|
strides[j] = tensor.dim[tensor.ndim - 2 - j] * strides[j + 1];
|
|
}
|
|
|
|
const int ident = 4;
|
|
uint64_t j = 0;
|
|
int broke = 1;
|
|
while (j < tensor.num_weights) {
|
|
int last = j + 1 == tensor.num_weights;
|
|
for (int k = 0; k < (int) tensor.ndim - 1; k++) {
|
|
if (j % strides[k] == 0) {
|
|
printf("%*s\n", k * ident, "[");
|
|
}
|
|
}
|
|
if (broke) {
|
|
printf("%*s", tensor.ndim * ident, "");
|
|
}
|
|
printf("%f%s", weights[j], last ? "" : ", ");
|
|
broke = 0;
|
|
j++;
|
|
for (int k = (int) tensor.ndim - 2; k >= 0; k--) {
|
|
if (j % strides[k] == 0) {
|
|
if (!broke) {
|
|
broke = 1;
|
|
printf("\n");
|
|
}
|
|
printf("%*s%s\n", k * ident, "]", last ? "" : ",");
|
|
}
|
|
}
|
|
if (!broke && j % 4 == 0) {
|
|
broke = 1;
|
|
printf("\n");
|
|
}
|
|
if (j == count) break;
|
|
}
|
|
if (!broke) printf("\n");
|
|
free(weights);
|
|
return;
|
|
}
|
|
|
|
/* ========================== 'compare' subcommand ========================== */
|
|
|
|
/* Given two tensors of the same length, return the average difference
|
|
* of their weights, in percentage.
|
|
*
|
|
* The difference is calculated like that: the average of the absolute values
|
|
* of all the weights in the two vectors is calculated. Then, for each set
|
|
* of corresponding weights, we calculate the difference, and the percentage
|
|
* according to the average value (100%). The function returns the average
|
|
* of the percentage of difference between all the pairs.
|
|
*
|
|
* Returns 1 on success, 0 if one or both the provided tensors can't be
|
|
* dequantized. */
|
|
int tensors_avg_diff(gguf_tensor *t1, gguf_tensor *t2, double *diff) {
|
|
float *weights1 = gguf_tensor_to_float(t1);
|
|
float *weights2 = gguf_tensor_to_float(t2);
|
|
if (weights1 == NULL || weights2 == NULL) {
|
|
free(weights1);
|
|
free(weights2);
|
|
return 0;
|
|
}
|
|
|
|
/* Compute the average magnitude of the weights. */
|
|
double tot_mag = 0;
|
|
for (uint64_t j = 0; j < t1->num_weights; j++) {
|
|
tot_mag += fabs(weights1[j]);
|
|
tot_mag += fabs(weights2[j]);
|
|
}
|
|
double avg_mag = tot_mag/(t1->num_weights*2);
|
|
|
|
/* Compute the average % difference of the weights. */
|
|
double tot_diff = 0;
|
|
for (uint64_t j = 0; j < t1->num_weights; j++)
|
|
tot_diff += fabs(weights1[j]-weights2[j]);
|
|
double avg_diff = tot_diff / t1->num_weights;
|
|
|
|
/* Multiply by 75 to normalize the difference of a
|
|
* random variable between -N and +N to 0 - 100% */
|
|
*diff = avg_diff / avg_mag * 75;
|
|
|
|
free(weights1);
|
|
free(weights2);
|
|
return 1;
|
|
}
|
|
|
|
void gguf_tools_compare(const char *file1, const char *file2) {
|
|
gguf_ctx *ctx1 = gguf_open(file1);
|
|
if (ctx1 == NULL) {
|
|
perror(file1);
|
|
exit(1);
|
|
}
|
|
|
|
gguf_ctx *ctx2 = gguf_open(file2);
|
|
if (ctx2 == NULL) {
|
|
perror(file2);
|
|
exit(1);
|
|
}
|
|
|
|
/* Skip all the key-value pairs. */
|
|
gguf_skip_key_values_section(ctx1);
|
|
|
|
/* For each tensor of the first net... */
|
|
gguf_tensor tensor1, tensor2;
|
|
while (gguf_get_tensor(ctx1,&tensor1)) {
|
|
gguf_skip_key_values_section(ctx2);
|
|
while (gguf_get_tensor(ctx2,&tensor2)) {
|
|
/* Search for a tensor with the same name. */
|
|
if (tensor2.namelen == tensor1.namelen &&
|
|
memcmp(tensor2.name,tensor1.name,tensor1.namelen) == 0)
|
|
{
|
|
printf("[%.*s]: ", (int)tensor1.namelen, tensor1.name);
|
|
fflush(stdout);
|
|
if (tensor1.num_weights != tensor2.num_weights) {
|
|
printf("size mismatch\n");
|
|
} else {
|
|
double diff;
|
|
if (tensors_avg_diff(&tensor1, &tensor2, &diff)) {
|
|
printf("avg weights difference: %f%%\n", diff);
|
|
} else {
|
|
printf("dequantization function missing...\n");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
gguf_rewind(ctx2);
|
|
}
|
|
}
|
|
|
|
/* ======================= Main and CLI options parsing ===================== */
|
|
|
|
void gguf_tools_usage(const char *progname) {
|
|
printf("Usage: %s <subcommand> [arguments...] [options...]\n"
|
|
"Subcommands:\n"
|
|
" show <filename> -- show GGUF model keys and tensors.\n"
|
|
" inspect-tensor <filename> <tensor-name> [count] -- show tensor weights.\n"
|
|
" compare <file1> <file2> -- avg weights diff for matching tensor names.\n"
|
|
" split-mixtral <ids...> mixtral.gguf out.gguf -- extract expert.\n"
|
|
"Options:\n"
|
|
" --verbose :With 'show', print full arrays (e.g. token lists)\n"
|
|
"Example:\n"
|
|
" split-mixtral 65230776370407150546470161412165 mixtral.gguf out.gguf\n"
|
|
, progname);
|
|
exit(1);
|
|
}
|
|
|
|
int main(int argc, char **argv) {
|
|
if (argc < 3) gguf_tools_usage(argv[0]);
|
|
|
|
/* Parse options before getting into subcommands parsing. */
|
|
for (int j = 1; j < argc; j++) {
|
|
/* Every time we find a an option, we try to parse it
|
|
* and set the used argv[] entires to NULL. Later we remove
|
|
* the NULL entries. In this way '--options' can be anywhere,
|
|
* making the tool simpler to use. */
|
|
if (!strcmp(argv[j],"--verbose")) {
|
|
argv[j] = NULL;
|
|
argc--;
|
|
Opt.verbose = 1;
|
|
}
|
|
}
|
|
|
|
/* Strip empty elements. */
|
|
for (int j = 1; j < argc; j++) {
|
|
if (argv[j] == NULL) {
|
|
memmove(argv+j, argv+j+1, sizeof(char*) * (argc-j));
|
|
}
|
|
}
|
|
|
|
if (!strcmp(argv[1],"show") && argc == 3) {
|
|
gguf_tools_show(argv[2]);
|
|
} else if (!strcmp(argv[1],"compare") && argc == 4) {
|
|
gguf_tools_compare(argv[2],argv[3]);
|
|
} else if (!strcmp(argv[1],"inspect-tensor") && (argc == 4 || argc == 5)) {
|
|
gguf_tools_inspect_weights(argv[2],argv[3],
|
|
argc == 5 ? atoi(argv[4]) : 0);
|
|
} else if (!strcmp(argv[1],"split-mixtral") && argc == 5) {
|
|
int experts[32];
|
|
size_t elen = strlen(argv[2]);
|
|
for (size_t j = 0; j < 32; j++) {
|
|
if (j < elen) {
|
|
experts[j] = argv[2][j] - '0';
|
|
if (experts[j] < 0 || experts[j] > 7) {
|
|
fprintf(stderr,"Invalid expert ID: %d\n", experts[j]);
|
|
exit(1);
|
|
}
|
|
} else {
|
|
/* If there aren't 32 digits in the input, use the last
|
|
* one repeated up to the last layer. */
|
|
experts[j] = j > 1 ? experts[j-1] : 0;
|
|
}
|
|
}
|
|
gguf_tools_split_mixtral(experts,argv[3],argv[4]);
|
|
} else {
|
|
gguf_tools_usage(argv[0]);
|
|
}
|
|
return 0;
|
|
}
|