From ed09c7190ea26f68faf9adba57feb3c7f404a26d Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sun, 4 Dec 2022 18:33:14 +0200 Subject: [PATCH] gpt : add support for gpt-jt + fix unicode support --- examples/gpt-2/convert-ckpt-to-ggml.py | 7 ++++++- examples/gpt-j/convert-h5-to-ggml.py | 9 +++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/examples/gpt-2/convert-ckpt-to-ggml.py b/examples/gpt-2/convert-ckpt-to-ggml.py index 09824a8..7ae4380 100644 --- a/examples/gpt-2/convert-ckpt-to-ggml.py +++ b/examples/gpt-2/convert-ckpt-to-ggml.py @@ -81,8 +81,9 @@ byte_encoder = bytes_to_unicode() byte_decoder = {v:k for k, v in byte_encoder.items()} fout.write(struct.pack("i", len(encoder))) + for key in encoder: - text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8') + text = bytearray([byte_decoder[c] for c in key]) fout.write(struct.pack("i", len(text))) fout.write(text) @@ -105,6 +106,10 @@ for name, shape in list_vars: print(" Converting to float16") data = data.astype(np.float16) ftype = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype = 0 # for efficiency - transpose the projection matrices if name[-13:] == "/mlp/c_proj/w": diff --git a/examples/gpt-j/convert-h5-to-ggml.py b/examples/gpt-j/convert-h5-to-ggml.py index a1efecb..310e60e 100644 --- a/examples/gpt-j/convert-h5-to-ggml.py +++ b/examples/gpt-j/convert-h5-to-ggml.py @@ -91,13 +91,14 @@ byte_encoder = bytes_to_unicode() byte_decoder = {v:k for k, v in byte_encoder.items()} fout.write(struct.pack("i", len(encoder) + len(encoder_added))) + for key in encoder: - text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8') + text = bytearray([byte_decoder[c] for c in key]) fout.write(struct.pack("i", len(text))) fout.write(text) for key in encoder_added: - text = bytearray([byte_decoder[c] for c in key]).decode('utf-8', errors='replace').encode('utf-8') + text = bytearray([byte_decoder[c] for c in key]) fout.write(struct.pack("i", len(text))) fout.write(text) @@ -119,6 +120,10 @@ for name in list_vars.keys(): print(" Converting to float16") data = data.astype(np.float16) ftype = 1 + else: + print(" Converting to float32") + data = data.astype(np.float32) + ftype = 0 # for efficiency - transpose these matrices: # "transformer.h.*.mlp.fc_in.weight