diff --git a/examples/gpt-2/convert-h5-to-ggml.py b/examples/gpt-2/convert-h5-to-ggml.py new file mode 100644 index 0000000..758fb59 --- /dev/null +++ b/examples/gpt-2/convert-h5-to-ggml.py @@ -0,0 +1,147 @@ +# Convert GPT-2 h5 transformer model to ggml format +# +# Load the model using GPT2Model. +# Iterate over all variables and write them to a binary file. +# +# For each variable, write the following: +# - Number of dimensions (int) +# - Name length (int) +# - Dimensions (int[n_dims]) +# - Name (char[name_length]) +# - Data (float[n_dims]) +# +# By default, the bigger matrices are converted to 16-bit floats. +# This can be disabled by adding the "use-f32" CLI argument. +# +# At the start of the ggml file we write the model parameters +# and vocabulary. +# + +import sys +import struct +import json +import torch +import numpy as np + +from transformers import GPT2Model + +# ref: https://github.com/openai/gpt-2/blob/master/src/encoder.py +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + +if len(sys.argv) < 2: + print("Usage: convert-h5-to-ggml.py dir-model [use-f32]\n") + sys.exit(1) + +# output in the same directory as the model +dir_model = sys.argv[1] +fname_out = sys.argv[1] + "/ggml-model.bin" + +with open(dir_model + "/vocab.json", "r") as f: + encoder = json.load(f) + +with open(dir_model + "/added_tokens.json", "r") as f: + encoder_added = json.load(f) + +with open(dir_model + "/config.json", "r") as f: + hparams = json.load(f) + +# use 16-bit or 32-bit floats +use_f16 = True +if len(sys.argv) > 2: + use_f16 = False + fname_out = sys.argv[1] + "/ggml-model-f32.bin" + +model = GPT2Model.from_pretrained(dir_model, low_cpu_mem_usage=True) +#print (model) + +list_vars = model.state_dict() +#print (list_vars) + +fout = open(fname_out, "wb") + +fout.write(struct.pack("i", 0x67676d6c)) # magic: ggml in hex +fout.write(struct.pack("i", hparams["vocab_size"])) +fout.write(struct.pack("i", hparams["n_positions"])) +fout.write(struct.pack("i", hparams["n_embd"])) +fout.write(struct.pack("i", hparams["n_head"])) +fout.write(struct.pack("i", hparams["n_layer"])) +#fout.write(struct.pack("i", hparams["rotary_dim"])) +fout.write(struct.pack("i", use_f16)) + +byte_encoder = bytes_to_unicode() +byte_decoder = {v:k for k, v in byte_encoder.items()} + +fout.write(struct.pack("i", len(encoder) + len(encoder_added))) + +for key in encoder: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for key in encoder_added: + text = bytearray([byte_decoder[c] for c in key]) + fout.write(struct.pack("i", len(text))) + fout.write(text) + +for name in list_vars.keys(): + data = list_vars[name].squeeze().numpy() + print("Processing variable: " + name + " with shape: ", data.shape) + + # we don't need these + if name.endswith("attn.masked_bias") or name.endswith(".attn.bias"): + print(" Skipping variable: " + name) + continue + + n_dims = len(data.shape); + + # ftype == 0 -> float32, ftype == 1 -> float16 + ftype = 0; + if use_f16: + if name[-7:] == ".weight" and n_dims == 2: + print(" Converting to float16") + data = data.astype(np.float16) + ftype = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype = 0 + + # for efficiency - transpose these matrices: + # "transformer.h.*.mlp.c_proj.weight + if name.endswith(".mlp.c_proj.weight"): + print(" Transposing") + data = data.transpose() + + # header + str = name.encode('utf-8') + fout.write(struct.pack("iii", n_dims, len(str), ftype)) + for i in range(n_dims): + fout.write(struct.pack("i", data.shape[n_dims - 1 - i])) + fout.write(str); + + # data + data.tofile(fout) + +fout.close() + +print("Done. Output file: " + fname_out) +print("") diff --git a/examples/gpt-2/main.cpp b/examples/gpt-2/main.cpp index 134a930..24b9972 100644 --- a/examples/gpt-2/main.cpp +++ b/examples/gpt-2/main.cpp @@ -208,11 +208,11 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & model.wpe = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, n_embd, n_ctx); // map by name - model.tensors["model/ln_f/g"] = model.ln_f_g; - model.tensors["model/ln_f/b"] = model.ln_f_b; + model.tensors["ln_f.weight"] = model.ln_f_g; + model.tensors["ln_f.bias"] = model.ln_f_b; - model.tensors["model/wte"] = model.wte; - model.tensors["model/wpe"] = model.wpe; + model.tensors["wte.weight"] = model.wte; + model.tensors["wpe.weight"] = model.wpe; for (int i = 0; i < n_layer; ++i) { auto & layer = model.layers[i]; @@ -236,23 +236,23 @@ bool gpt2_model_load(const std::string & fname, gpt2_model & model, gpt_vocab & layer.c_mlp_proj_b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd); // map by name - model.tensors["model/h" + std::to_string(i) + "/ln_1/g"] = layer.ln_1_g; - model.tensors["model/h" + std::to_string(i) + "/ln_1/b"] = layer.ln_1_b; + model.tensors["h." + std::to_string(i) + ".ln_1.weight"] = layer.ln_1_g; + model.tensors["h." + std::to_string(i) + ".ln_1.bias"] = layer.ln_1_b; - model.tensors["model/h" + std::to_string(i) + "/ln_2/g"] = layer.ln_2_g; - model.tensors["model/h" + std::to_string(i) + "/ln_2/b"] = layer.ln_2_b; + model.tensors["h." + std::to_string(i) + ".ln_2.weight"] = layer.ln_2_g; + model.tensors["h." + std::to_string(i) + ".ln_2.bias"] = layer.ln_2_b; - model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/w"] = layer.c_attn_attn_w; - model.tensors["model/h" + std::to_string(i) + "/attn/c_attn/b"] = layer.c_attn_attn_b; + model.tensors["h." + std::to_string(i) + ".attn.c_attn.weight"] = layer.c_attn_attn_w; + model.tensors["h." + std::to_string(i) + ".attn.c_attn.bias"] = layer.c_attn_attn_b; - model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/w"] = layer.c_attn_proj_w; - model.tensors["model/h" + std::to_string(i) + "/attn/c_proj/b"] = layer.c_attn_proj_b; + model.tensors["h." + std::to_string(i) + ".attn.c_proj.weight"] = layer.c_attn_proj_w; + model.tensors["h." + std::to_string(i) + ".attn.c_proj.bias"] = layer.c_attn_proj_b; - model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/w"] = layer.c_mlp_fc_w; - model.tensors["model/h" + std::to_string(i) + "/mlp/c_fc/b"] = layer.c_mlp_fc_b; + model.tensors["h." + std::to_string(i) + ".mlp.c_fc.weight"] = layer.c_mlp_fc_w; + model.tensors["h." + std::to_string(i) + ".mlp.c_fc.bias"] = layer.c_mlp_fc_b; - model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/w"] = layer.c_mlp_proj_w_trans; - model.tensors["model/h" + std::to_string(i) + "/mlp/c_proj/b"] = layer.c_mlp_proj_b; + model.tensors["h." + std::to_string(i) + ".mlp.c_proj.weight"] = layer.c_mlp_proj_w_trans; + model.tensors["h." + std::to_string(i) + ".mlp.c_proj.bias"] = layer.c_mlp_proj_b; } }