From 7211862c943273fc8ce4b7fdf4c04f9821b7b591 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Sat, 11 Mar 2023 12:26:16 +0200 Subject: [PATCH] Update Makefile var + add comment --- Makefile | 6 +++--- README.md | 8 +++++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 862d02b..8388c29 100644 --- a/Makefile +++ b/Makefile @@ -125,18 +125,18 @@ ifneq ($(filter ppc64%,$(UNAME_M)),) CXXFLAGS += -std=c++23 -DGGML_BIG_ENDIAN endif endif -ifndef WHISPER_NO_ACCELERATE +ifndef LLAMA_NO_ACCELERATE # Mac M1 - include Accelerate framework ifeq ($(UNAME_S),Darwin) CFLAGS += -DGGML_USE_ACCELERATE LDFLAGS += -framework Accelerate endif endif -ifdef WHISPER_OPENBLAS +ifdef LLAMA_OPENBLAS CFLAGS += -DGGML_USE_OPENBLAS -I/usr/local/include/openblas LDFLAGS += -lopenblas endif -ifdef WHISPER_GPROF +ifdef LLAMA_GPROF CFLAGS += -pg CXXFLAGS += -pg endif diff --git a/README.md b/README.md index 2486d4d..26f8cb5 100644 --- a/README.md +++ b/README.md @@ -180,5 +180,11 @@ When running the larger models, make sure you have enough disk space to store al In general, it seems to work, but I think it fails for unicode character support. Hopefully, someone can help with that - I don't know yet how much the quantization affects the quality of the generated text - Probably the token sampling can be improved -- x86 quantization support [not yet ready](https://github.com/ggerganov/ggml/pull/27). Basically, you want to run this on Apple Silicon. For now, on Linux and Windows you can use the F16 `ggml-model-f16.bin` model, but it will be much slower. +- x86 quantization support [not yet ready](https://github.com/ggerganov/ggml/pull/27). Basically, you want to run this + on Apple Silicon. For now, on Linux and Windows you can use the F16 `ggml-model-f16.bin` model, but it will be much + slower. +- The Accelerate framework is actually currently unused since I found that for tensors shapes typical for the Decoder, + there is no benefit compared to the ARM_NEON intrinsics implementation. Of course, it's possible that I simlpy don't + know how to utilize it properly. But in any case, you can even disable it with `LLAMA_NO_ACCELERATE=1 make` and the + performance will be the same, since no BLAS calls are invoked by the current implementation