diff --git a/misc/ollama/Makefile b/misc/ollama/Makefile index 1995935ddfae..be457b2db084 100644 --- a/misc/ollama/Makefile +++ b/misc/ollama/Makefile @@ -1,52 +1,58 @@ PORTNAME= ollama DISTVERSIONPREFIX= v -DISTVERSION= 0.3.4 -PORTREVISION= 4 +DISTVERSION= 0.3.6 CATEGORIES= misc # machine-learning MAINTAINER= yuri@FreeBSD.org COMMENT= Run Llama 2, Mistral, and other large language models WWW= https://ollama.com/ LICENSE= MIT LICENSE_FILE= ${WRKSRC}/LICENSE ONLY_FOR_ARCHS= amd64 ONLY_FOR_ARCHS_REASON= bundled patched llama-cpp is placed into the arch-specific path BUILD_DEPENDS= bash:shells/bash \ cmake:devel/cmake-core \ glslc:graphics/shaderc \ vulkan-headers>0:graphics/vulkan-headers LIB_DEPENDS= libvulkan.so:graphics/vulkan-loader USES= go:1.22,modules pkgconfig CONFLICTS_BUILD= llama-cpp GO_MODULE= github.com/${PORTNAME}/${PORTNAME} GO_TARGET= . USE_GITHUB= nodefault -GH_TUPLE= ggerganov:llama.cpp:6eeaeba:llama_cpp/llm/llama.cpp +GH_TUPLE= ggerganov:llama.cpp:1e6f6554aa11fa10160a5fda689e736c3c34169f:llama_cpp/llm/llama.cpp \ + blabber:go-freebsd-sysctl:503969f:go_sysctl/vendor.x/github.com/blabber/go-freebsd-sysctl MAKE_ENV= PATH=${PATH}:${WRKSRC}/llm/build/bsd/x86_64_static/bin # workaround to find vulkan-shaders-gen -PLIST_FILES= bin/${PORTNAME} +PLIST_FILES= bin/${PORTNAME} \ + bin/ollama-limit-layers post-patch: # workaround for https://github.com/ollama/ollama/issues/6259 (use of extenral libllama.so) @${REINPLACE_CMD} \ -e '\ s| llama | llama ${LOCALBASE}/lib/libvulkan.so omp pthread |; \ s| llama | ${WRKSRC}/llm/build/bsd/x86_64_static/src/libllama.a |; \ s| ggml | ${WRKSRC}/llm/build/bsd/x86_64_static/ggml/src/libggml.a |; \ ' \ ${WRKSRC}/llm/ext_server/CMakeLists.txt + # move vendor.x to vendor + @(cd ${WRKSRC}/vendor.x && ${TAR} cf - .) | (cd ${WRKSRC}/vendor && ${TAR} xf -) pre-build: @${CP} ${WRKSRC}/app/store/store_linux.go ${WRKSRC}/app/store/store_bsd.go @cd ${GO_WRKSRC} && \ ${SETENVI} ${WRK_ENV} ${MAKE_ENV} ${GO_ENV} GOMAXPROCS=${MAKE_JOBS_NUMBER} GOPROXY=off ${GO_CMD} generate ${GO_BUILDFLAGS} \ ./... +post-install: # pending https://github.com/ollama/ollama/issues/6407 + ${INSTALL_SCRIPT} ${FILESDIR}/ollama-limit-layers ${STAGEDIR}${PREFIX}/bin + .include diff --git a/misc/ollama/distinfo b/misc/ollama/distinfo index c436195aa415..86cae5d113d3 100644 --- a/misc/ollama/distinfo +++ b/misc/ollama/distinfo @@ -1,7 +1,9 @@ -TIMESTAMP = 1722929347 -SHA256 (go/misc_ollama/ollama-v0.3.4/v0.3.4.mod) = ec7b6ce8fd820503671d269fd43ea5ae7275b1ff095c77d6422ac8540ccb410b -SIZE (go/misc_ollama/ollama-v0.3.4/v0.3.4.mod) = 2992 -SHA256 (go/misc_ollama/ollama-v0.3.4/v0.3.4.zip) = d9bc3d3901272f8fbc9b57ce1af1caa2209127978bb5f72257b6a3fb84161969 -SIZE (go/misc_ollama/ollama-v0.3.4/v0.3.4.zip) = 1832031 -SHA256 (go/misc_ollama/ollama-v0.3.4/ggerganov-llama.cpp-6eeaeba_GH0.tar.gz) = f749f494dc5e672926179524cf28c834151edba5acae18e132c954df76ffdf33 -SIZE (go/misc_ollama/ollama-v0.3.4/ggerganov-llama.cpp-6eeaeba_GH0.tar.gz) = 19003544 +TIMESTAMP = 1724010094 +SHA256 (go/misc_ollama/ollama-v0.3.6/v0.3.6.mod) = 16c078d8f0b29f84598fb04e3979acf86da41eb41bf4ff8363548e490f38b54e +SIZE (go/misc_ollama/ollama-v0.3.6/v0.3.6.mod) = 2992 +SHA256 (go/misc_ollama/ollama-v0.3.6/v0.3.6.zip) = 94d2376c9555dd03a76cb093f3213e8155811874db7eab6aff2941d0e75dce07 +SIZE (go/misc_ollama/ollama-v0.3.6/v0.3.6.zip) = 1842735 +SHA256 (go/misc_ollama/ollama-v0.3.6/ggerganov-llama.cpp-1e6f6554aa11fa10160a5fda689e736c3c34169f_GH0.tar.gz) = b2b3137f734fc9a202fac710986f1de837e0ae69a0b532d4cbadb90748f4cb73 +SIZE (go/misc_ollama/ollama-v0.3.6/ggerganov-llama.cpp-1e6f6554aa11fa10160a5fda689e736c3c34169f_GH0.tar.gz) = 19016896 +SHA256 (go/misc_ollama/ollama-v0.3.6/blabber-go-freebsd-sysctl-503969f_GH0.tar.gz) = 1f497780d88f6f056b8d8f59b6aa129566c8041d16d1cda89f212accf88f3165 +SIZE (go/misc_ollama/ollama-v0.3.6/blabber-go-freebsd-sysctl-503969f_GH0.tar.gz) = 2117 diff --git a/misc/ollama/files/ollama-limit-layers b/misc/ollama/files/ollama-limit-layers new file mode 100755 index 000000000000..6713a9f9b02b --- /dev/null +++ b/misc/ollama/files/ollama-limit-layers @@ -0,0 +1,13 @@ +#!/bin/sh + +MODEL=$1 +NUM_GPU=$2 + +if [ -z "$MODEL" ] || [ -z "$NUM_GPU" ]; then + echo "Usage: $0 " +fi + +ollama show --modelfile $MODEL > Modelfile && +echo "PARAMETER num_gpu $NUM_GPU" >> Modelfile && +ollama create "$MODEL-num_gpu$NUM_GPU" -f Modelfile && +echo "model variant $MODEL-num_gpu$NUM_GPU was created" diff --git a/misc/ollama/files/patch-FreeBSD-compatibility b/misc/ollama/files/patch-FreeBSD-compatibility index ca206363f042..046ca9b3417c 100644 --- a/misc/ollama/files/patch-FreeBSD-compatibility +++ b/misc/ollama/files/patch-FreeBSD-compatibility @@ -1,256 +1,277 @@ -- patch based on https://github.com/ollama/ollama/issues/1102#issuecomment-2270042340 new file mode 100644 ---- gpu/gpu_bsd.go.orig 2024-08-06 07:41:26 UTC +--- gpu/gpu_bsd.go.orig 2024-08-18 20:03:12 UTC +++ gpu/gpu_bsd.go -@@ -0,0 +1,101 @@ +@@ -0,0 +1,122 @@ +//go:build dragonfly || freebsd || netbsd || openbsd + +package gpu + +import "github.com/ollama/ollama/format" ++//import sysctl "github.com/lorenzosaino/go-sysctl" // sysctl: this is Linux-only, see https://github.com/lorenzosaino/go-sysctl/issues/7 ++import sysctl "github.com/blabber/go-freebsd-sysctl/sysctl" // sysctl: this is FreeBSD-only basic library ++import ( ++ "log/slog" ++) + +/* +#cgo CFLAGS: -I/usr/local/include +#cgo LDFLAGS: -L/usr/local/lib -lvulkan + +#include +#include +#include + +bool hasVulkanSupport(uint64_t *memSize) { + VkInstance instance; + + VkApplicationInfo appInfo = { VK_STRUCTURE_TYPE_APPLICATION_INFO }; + appInfo.pApplicationName = "Ollama"; + appInfo.apiVersion = VK_API_VERSION_1_0; + + VkInstanceCreateInfo createInfo = { VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO }; + createInfo.pApplicationInfo = &appInfo; + + // Create a Vulkan instance + if (vkCreateInstance(&createInfo, NULL, &instance) != VK_SUCCESS) + return false; + + // Fetch the first physical Vulkan device. Note that numDevices is overwritten with the number of devices found + uint32_t numDevices = 1; + VkPhysicalDevice device; + vkEnumeratePhysicalDevices(instance, &numDevices, &device); + if (numDevices == 0) { + vkDestroyInstance(instance, NULL); + return false; + } + + // Fetch the memory information for this device. + VkPhysicalDeviceMemoryProperties memProperties; + vkGetPhysicalDeviceMemoryProperties(device, &memProperties); + + // Add up all the heaps. + VkDeviceSize totalMemory = 0; + for (uint32_t i = 0; i < memProperties.memoryHeapCount; ++i) { + if (memProperties.memoryHeaps[i].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) { + *memSize += memProperties.memoryHeaps[i].size; + } + } + + vkDestroyInstance(instance, NULL); + return true; +} +*/ +import "C" + +func GetGPUInfo() GpuInfoList { + var gpuMem C.uint64_t + if C.hasVulkanSupport(&gpuMem) { + // Vulkan supported + return []GpuInfo{ + { + Library: "vulkan", + ID: "0", + MinimumMemory: 512 * format.MebiByte, + memInfo: memInfo{ + FreeMemory: uint64(gpuMem), + TotalMemory: uint64(gpuMem), + }, + }, + } + } + + // CPU fallback + cpuMem, _ := GetCPUMem() + return []GpuInfo{ + { + Library: "cpu", + memInfo: cpuMem, + }, + } +} + +func GetCPUInfo() GpuInfoList { + mem, _ := GetCPUMem() + return []GpuInfo{ + { + Library: "cpu", + Variant: GetCPUCapability(), + memInfo: mem, + }, + } +} + +func GetCPUMem() (memInfo, error) { -+ size := C.sysconf(C._SC_PHYS_PAGES) * C.sysconf(C._SC_PAGE_SIZE) -+ return memInfo{TotalMemory: uint64(size)}, nil ++ // all involved sysctl variables ++ sysctl_vm_page_size, _ := sysctl.GetInt64("vm.stats.vm.v_page_size") // memory page size ++ sysctl_hw_physmem, _ := sysctl.GetInt64("hw.physmem") // physical memory in bytes ++ sysctl_vm_free_count, _ := sysctl.GetInt64("vm.stats.vm.v_free_count") // free page count ++ sysctl_vm_swap_total, _ := sysctl.GetInt64("vm.swap_total") // total swap size in bytes ++ ++ // individual values ++ total_memory := uint64(sysctl_hw_physmem) ++ free_memory := uint64(sysctl_vm_free_count) * uint64(sysctl_vm_page_size) ++ free_swap := uint64(sysctl_vm_swap_total) // wrong to use the total swap size here, should be vm.swap_free, see https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=280909 ++ ++ slog.Debug("gpu_bsd.go::GetCPUMem::GetCPUMem", "total_memory", total_memory, "free_memory", free_memory, "free_swap", free_swap) ++ ++ return memInfo{ ++ TotalMemory: uint64(total_memory), ++ FreeMemory: uint64(free_memory), ++ FreeSwap: uint64(free_swap), ++ }, nil +} + +func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) { + return "", "" +} --- gpu/gpu_test.go.orig 1979-11-30 08:00:00 UTC +++ gpu/gpu_test.go @@ -11,7 +11,7 @@ func TestBasicGetGPUInfo(t *testing.T) { func TestBasicGetGPUInfo(t *testing.T) { info := GetGPUInfo() assert.NotEmpty(t, len(info)) - assert.Contains(t, "cuda rocm cpu metal", info[0].Library) + assert.Contains(t, "cuda rocm cpu metal vulkan", info[0].Library) if info[0].Library != "cpu" { assert.Greater(t, info[0].TotalMemory, uint64(0)) assert.Greater(t, info[0].FreeMemory, uint64(0)) @@ -24,6 +24,8 @@ func TestCPUMemInfo(t *testing.T) { switch runtime.GOOS { case "darwin": t.Skip("CPU memory not populated on darwin") + case "dragonfly", "freebsd", "netbsd", "openbsd": + t.Skip("CPU memory is not populated on *BSD") case "linux", "windows": assert.Greater(t, info.TotalMemory, uint64(0)) assert.Greater(t, info.FreeMemory, uint64(0)) --- llm/generate/gen_bsd.sh.orig 2024-08-06 16:29:05 UTC +++ llm/generate/gen_bsd.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# This script is intended to run inside the go generate +# working directory must be ./llm/generate/ + +set -ex +set -o pipefail +echo "Starting BSD generate script" +. $(dirname $0)/gen_common.sh +init_vars +#git_module_setup +apply_patches + +COMMON_BSD_DEFS="-DCMAKE_SYSTEM_NAME=$(uname -s)" +CMAKE_TARGETS="--target llama --target ggml" + +COMMON_CPU_DEFS="${COMMON_BSD_DEFS} -DCMAKE_SYSTEM_PROCESSOR=${ARCH} -DBUILD_SHARED_LIBS=off" + +# Static build for linking into the Go binary +init_vars +CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_VULKAN=on -DGGML_ACCELERATE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}" +BUILD_DIR="../build/bsd/${ARCH}_static" +echo "Building static library" +build + +init_vars +CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}" +BUILD_DIR="../build/bsd/${ARCH}/cpu" +echo "Building LCD CPU" +build +compress + +init_vars +CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=on -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off ${CMAKE_DEFS}" +BUILD_DIR="../build/bsd/${ARCH}/cpu_avx" +echo "Building AVX CPU" +build +compress + +init_vars +CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_AVX=on -DGGML_AVX2=on -DGGML_AVX512=off -DGGML_FMA=on -DGGML_F16C=on ${CMAKE_DEFS}" +BUILD_DIR="../build/bsd/${ARCH}/cpu_avx2" +echo "Building AVX2 CPU" +build +compress + +init_vars +CMAKE_DEFS="${COMMON_CPU_DEFS} -DGGML_VULKAN=on ${CMAKE_DEFS}" +BUILD_DIR="../build/bsd/${ARCH}/vulkan" +echo "Building Vulkan GPU" +build +compress + +cleanup +echo "go generate completed. LLM runners: $(cd ${BUILD_DIR}/..; echo *)" --- llm/generate/generate_bsd.go.orig 2024-08-06 07:41:26 UTC +++ llm/generate/generate_bsd.go @@ -0,0 +1,5 @@ +//go:build dragonfly || freebsd || netbsd || openbsd + +package generate + +//go:generate bash ./gen_bsd.sh --- llm/llm.go.orig 1979-11-30 08:00:00 UTC +++ llm/llm.go @@ -8,6 +8,10 @@ package llm // #cgo windows,arm64 LDFLAGS: -static-libstdc++ -static-libgcc -static -L${SRCDIR}/build/windows/arm64_static -L${SRCDIR}/build/windows/arm64_static/src -L${SRCDIR}/build/windows/arm64_static/ggml/src // #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/build/linux/x86_64_static -L${SRCDIR}/build/linux/x86_64_static/src -L${SRCDIR}/build/linux/x86_64_static/ggml/src // #cgo linux,arm64 LDFLAGS: -L${SRCDIR}/build/linux/arm64_static -L${SRCDIR}/build/linux/arm64_static/src -L${SRCDIR}/build/linux/arm64_static/ggml/src +// #cgo dragonfly,amd64 LDFLAGS: ${SRCDIR}/build/bsd/x86_64_static/src/libllama.a -lstdc++ -lm +// #cgo freebsd,amd64 LDFLAGS: -L${SRCDIR}/build/bsd/x86_64_static/src -lllama -L${SRCDIR}/build/bsd/x86_64_static/ggml/src -lggml -lstdc++ -lm -lomp +// #cgo netbsd,amd64 LDFLAGS: ${SRCDIR}/build/bsd/x86_64_static/src/libllama.a -lstdc++ -lm +// #cgo openbsd,amd64 LDFLAGS: ${SRCDIR}/build/bsd/x86_64_static/src/libllama.a -lstdc++ -lm // #include // #include "llama.h" import "C" --- llm/llm_bsd.go.orig 2024-08-06 07:41:26 UTC +++ llm/llm_bsd.go @@ -0,0 +1,13 @@ +//go:build dragonfly || freebsd || netbsd || openbsd + +package llm + +import ( + "embed" + "syscall" +) + +//go:embed build/bsd/*/*/bin/* +var libEmbed embed.FS + +var LlamaServerSysProcAttr = &syscall.SysProcAttr{} --- scripts/build_bsd.sh.orig 2024-08-06 07:41:26 UTC +++ scripts/build_bsd.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +set -e + +case "$(uname -s)" in + DragonFly) + ;; + FreeBSD) + ;; + NetBSD) + ;; + OpenBSD) + ;; + *) + echo "$(uname -s) is not supported" + exit 1 + ;; +esac + +export VERSION=${VERSION:-$(git describe --tags --first-parent --abbrev=7 --long --dirty --always | sed -e "s/^v//g")} +export GOFLAGS="'-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=$VERSION\" \"-X=github.com/ollama/ollama/server.mode=release\"'" + +mkdir -p dist +rm -rf llm/llama.cpp/build + +go generate ./... +CGO_ENABLED=1 go build -trimpath -o dist/ollama-bsd --- scripts/build_freebsd.sh.orig 2024-08-06 07:41:26 UTC +++ scripts/build_freebsd.sh @@ -0,0 +1 @@ +build_bsd.sh \ No newline at end of file diff --git a/misc/ollama/files/patch-llm_llama.cpp_ggml_src_vulkan-shaders_CMakeLists.txt b/misc/ollama/files/patch-llm_llama.cpp_ggml_src_vulkan-shaders_CMakeLists.txt index 3ada7c2debc2..a9ef6bec163e 100644 --- a/misc/ollama/files/patch-llm_llama.cpp_ggml_src_vulkan-shaders_CMakeLists.txt +++ b/misc/ollama/files/patch-llm_llama.cpp_ggml_src_vulkan-shaders_CMakeLists.txt @@ -1,9 +1,10 @@ ---- llm/llama.cpp/ggml/src/vulkan-shaders/CMakeLists.txt.orig 2024-08-08 21:55:59 UTC +--- llm/llama.cpp/ggml/src/vulkan-shaders/CMakeLists.txt.orig 2024-08-06 15:33:39 UTC +++ llm/llama.cpp/ggml/src/vulkan-shaders/CMakeLists.txt -@@ -1,5 +1,6 @@ add_executable(${TARGET} vulkan-shaders-gen.cpp) +@@ -2,6 +2,7 @@ add_executable(${TARGET} vulkan-shaders-gen.cpp) set(TARGET vulkan-shaders-gen) add_executable(${TARGET} vulkan-shaders-gen.cpp) +target_link_libraries(${TARGET} PRIVATE pthread) install(TARGETS ${TARGET} RUNTIME) target_compile_features(${TARGET} PRIVATE cxx_std_11) + target_link_libraries(vulkan-shaders-gen PUBLIC Threads::Threads) diff --git a/misc/ollama/pkg-message b/misc/ollama/pkg-message index 90096ac82cef..95b0d4e58448 100644 --- a/misc/ollama/pkg-message +++ b/misc/ollama/pkg-message @@ -1,24 +1,31 @@ [ { type: install message: <