Refactor package structure

Makefile

@@ -11,12 +11,11 @@ MAKEFLAGS += --no-builtin-rules
 include build/config.mk
 include build/rules.mk
 
+include third_party/BUILD.mk
 include llamafile/BUILD.mk
 include llama.cpp/BUILD.mk
 include stable-diffusion.cpp/BUILD.mk
 include whisper.cpp/BUILD.mk
-include double-conversion/BUILD.mk
-include stb/BUILD.mk
 
 # the root package is `o//` by default
 # building a package also builds its sub-packages
@@ -25,7 +24,7 @@ o/$(MODE)/:	o/$(MODE)/llamafile					\
 		o/$(MODE)/llama.cpp					\
 		o/$(MODE)/stable-diffusion.cpp				\
 		o/$(MODE)/whisper.cpp					\
-		o/$(MODE)/stb						\
+		o/$(MODE)/third_party					\
 		o/$(MODE)/depend.test
 
 # for installing to `make PREFIX=/usr/local`

llama.cpp/llava/BUILD.mk

@@ -16,7 +16,7 @@ o/$(MODE)/llama.cpp/llava/llava-quantize:				\
 		o/$(MODE)/llama.cpp/llava/llava-quantize.1.asc.zip.o	\
 		o/$(MODE)/llama.cpp/llava/llava.a			\
 		o/$(MODE)/llama.cpp/llama.cpp.a				\
-		o/$(MODE)/stb/stb.a
+		o/$(MODE)/third_party/stb/stb.a
 
 $(LLAMA_CPP_LLAVA_OBJS): llama.cpp/llava/BUILD.mk
 

llama.cpp/llava/clip.cpp

@@ -13,7 +13,7 @@
 #include "llama.cpp/ggml-metal.h"
 #include "llama.cpp/string.h"
 
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
 
 #include <cassert>
 #include <cmath>

llama.cpp/main/BUILD.mk

@@ -14,7 +14,8 @@ o/$(MODE)/llama.cpp/main/main:					\
 		o/$(MODE)/llama.cpp/server/server.a		\
 		o/$(MODE)/llama.cpp/llava/llava.a		\
 		o/$(MODE)/llama.cpp/llama.cpp.a			\
-		o/$(MODE)/stb/stb.a				\
+		o/$(MODE)/llamafile/highlight/highlight.a	\
+		o/$(MODE)/third_party/stb/stb.a			\
 		o/$(MODE)/llama.cpp/main/main.1.asc.zip.o	\
 		$(LLAMA_CPP_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
 

llama.cpp/server/server.cpp

@@ -5,7 +5,7 @@
 #include "llama.cpp/llama.h"
 #include "llama.cpp/grammar-parser.h"
 #include "llama.cpp/llava/llava.h"
-#include "stb/stb_image.h"
+#include "third_party/stb/stb_image.h"
 #include "utils.h"
 #include "oai.h"
 #include "llamafile/micros.h"

llamafile/BUILD.mk

@@ -9,23 +9,21 @@ LLAMAFILE_INCS = $(filter %.inc,$(LLAMAFILE_FILES))
 LLAMAFILE_SRCS_C = $(filter %.c,$(LLAMAFILE_FILES))
 LLAMAFILE_SRCS_CU = $(filter %.cu,$(LLAMAFILE_FILES))
 LLAMAFILE_SRCS_CPP = $(filter %.cpp,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS_GPERF = $(filter %.gperf,$(LLAMAFILE_FILES))
-LLAMAFILE_SRCS_GPERF_C = $(LLAMAFILE_SRCS_GPERF:%.gperf=o/$(MODE)/%.c)
-LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU) $(LLAMAFILE_SRCS_GPERF)
+LLAMAFILE_SRCS = $(LLAMAFILE_SRCS_C) $(LLAMAFILE_SRCS_CPP) $(LLAMAFILE_SRCS_CU)
 LLAMAFILE_DOCS = $(filter %.1,$(LLAMAFILE_FILES))
 
 LLAMAFILE_OBJS :=					\
 	$(LLAMAFILE_SRCS_C:%.c=o/$(MODE)/%.o)		\
 	$(LLAMAFILE_SRCS_CPP:%.cpp=o/$(MODE)/%.o)	\
 	$(LLAMAFILE_FILES:%=o/$(MODE)/%.zip.o)		\
-	$(LLAMAFILE_SRCS_GPERF_C:%.c=%.o)		\
 
 $(LLAMAFILE_OBJS): private CCFLAGS += -g
 
 # this executable defines its own malloc(), free(), etc.
 # therefore we want to avoid it going inside the .a file
 LLAMAFILE_OBJS := $(filter-out o/$(MODE)/llamafile/zipalign.o,$(LLAMAFILE_OBJS))
 
+include llamafile/highlight/BUILD.mk
 include llamafile/server/BUILD.mk
 
 o/$(MODE)/llamafile/zipalign:				\
@@ -57,9 +55,6 @@ o/$(MODE)/llamafile:						\
 		o/$(MODE)/llamafile/addnl			\
 		o/$(MODE)/llamafile/high			\
 		o/$(MODE)/llamafile/datauri_test.runs		\
-		o/$(MODE)/llamafile/highlight_c_test.runs	\
-		o/$(MODE)/llamafile/highlight_python_test.runs	\
-		o/$(MODE)/llamafile/highlight_test.runs		\
 		o/$(MODE)/llamafile/parse_cidr_test.runs	\
 		o/$(MODE)/llamafile/pool_cancel_test.runs	\
 		o/$(MODE)/llamafile/pool_test.runs		\
@@ -170,34 +165,15 @@ o/$(MODE)/llamafile/pool_test:				\
 		o/$(MODE)/llamafile/crash.o		\
 		o/$(MODE)/llamafile/pool.o		\
 
-o/$(MODE)/llamafile/highlight_test:			\
-		o/$(MODE)/llamafile/highlight_test.o	\
-		o/$(MODE)/llama.cpp/llama.cpp.a		\
-
-o/$(MODE)/llamafile/highlight_c_test:				\
-		o/$(MODE)/llamafile/highlight_c_test.o		\
-		o/$(MODE)/llamafile/highlight_c.o		\
-		o/$(MODE)/llamafile/is_keyword_c.o		\
-		o/$(MODE)/llamafile/is_keyword_c_constant.o	\
-		o/$(MODE)/llamafile/is_keyword_c_type.o		\
-		o/$(MODE)/llamafile/is_keyword_c_pod.o		\
-		o/$(MODE)/llamafile/is_keyword_cpp.o		\
-
-o/$(MODE)/llamafile/highlight_python_test:				\
-		o/$(MODE)/llamafile/highlight_python_test.o		\
-		o/$(MODE)/llamafile/highlight_python.o			\
-		o/$(MODE)/llamafile/is_keyword_python.o			\
-		o/$(MODE)/llamafile/is_keyword_python_builtin.o		\
-		o/$(MODE)/llamafile/is_keyword_python_constant.o	\
-
 o/$(MODE)/llamafile/datauri_test:			\
 		o/$(MODE)/llamafile/datauri_test.o	\
 		o/$(MODE)/llama.cpp/llama.cpp.a		\
-		o/$(MODE)/stb/stb.a			\
+		o/$(MODE)/third_party/stb/stb.a		\
 
-o/$(MODE)/llamafile/high:				\
-		o/$(MODE)/llamafile/high.o		\
-		o/$(MODE)/llama.cpp/llama.cpp.a		\
+o/$(MODE)/llamafile/high:					\
+		o/$(MODE)/llamafile/high.o			\
+		o/$(MODE)/llamafile/highlight/highlight.a	\
+		o/$(MODE)/llama.cpp/llama.cpp.a			\
 
 o/$(MODE)/llamafile/hex2xterm:				\
 		o/$(MODE)/llamafile/hex2xterm.o		\
@@ -257,5 +233,4 @@ o/$(MODE)/llamafile/pick_a_warp_kernel.o: private CFLAGS += -fopenmp
 
 .PHONY: o/$(MODE)/llamafile/check
 o/$(MODE)/llamafile/check:				\
-		$(LLAMAFILE_SRCS_GPERF_C)		\
 		o/$(MODE)/llamafile/tinyblas_test.runs

llamafile/chatbot_repl.cpp

@@ -26,7 +26,7 @@
 #include "llama.cpp/llama.h"
 #include "llamafile/bestline.h"
 #include "llamafile/color.h"
-#include "llamafile/highlight.h"
+#include "llamafile/highlight/highlight.h"
 #include "llamafile/llama.h"
 
 namespace lf {

llamafile/high.cpp

@@ -15,7 +15,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "highlight.h"
+#include "llamafile/highlight/highlight.h"
 #include "string.h"
 #include <fcntl.h>
 #include <stdio.h>