Java components for machine learning

diff --git a/rebuild/Makefile b/rebuild/Makefile
index 243c3e0a1cf85641d2c620e62ab3e5c753535da7..9b4313699b4f617a9aac1a6d5e38fef8e3216c97 100644 (file)
--- a/rebuild/Makefile
+++ b/rebuild/Makefile
@@ -2,12 +2,17 @@
 
 all:
        make -C org.argeo.tp.sys
+       make -C org.argeo.tp.ml
+
 install:
        make -C org.argeo.tp.sys install
+       make -C org.argeo.tp.ml install
        
 uninstall:
        make -C org.argeo.tp.sys uninstall
+       make -C org.argeo.tp.ml uninstall
        
 clean:
        make -C org.argeo.tp.sys clean
+       make -C org.argeo.tp.ml clean
        
\ No newline at end of file

diff --git a/rebuild/org.argeo.tp.ml/Makefile b/rebuild/org.argeo.tp.ml/Makefile
index 41203c5f146dcecbe22cba0e3fdc5f84b3b171f8..6e06256e5c6755eaf1ca1ddc8cfe8535c5123052 100644 (file)
--- a/rebuild/org.argeo.tp.ml/Makefile
+++ b/rebuild/org.argeo.tp.ml/Makefile
@@ -5,6 +5,9 @@ export NO_SDK_LEGAL := true
 
 A2_CATEGORY = org.argeo.tp.ml
 
+JLAMA_JNI_SRC=jni/com_github_tjake_jlama
+JLAMA_TARGET_EXEC=libjlama.so
+
 ## FIXME - DON'T FORGET TO UPDATE THE VERSION IN THE RELATED bnd.bnd FILE!
 JLLAMA_BRANCH=3.3
 JLLAMA_VERSION=$(JLLAMA_BRANCH).0
@@ -12,9 +15,13 @@ JLLAMA_VERSION=$(JLLAMA_BRANCH).0
 LLAMA3_COMMIT=5f602500beb593dce2726df7e2dd08b0803aa86b
 LLAMA3_VERSION=0.1.0
 
-all: retrieve-jllama retrieve-llama3 osgi
+JLAMA_BRANCH=0.2
+JLAMA_VERSION=$(JLAMA_BRANCH).1
+
+all: retrieve-jllama retrieve-llama3 retrieve-jlama osgi jni-jlama
        mkdir -p $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
        mv $(SDK_BUILD_BASE)/a2/$(A2_CATEGORY)/de.kherud.llama.$(JLLAMA_BRANCH).jar $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
+       mv $(SDK_BUILD_BASE)/a2/$(A2_CATEGORY)/com.github.tjake.jlama.native.$(JLAMA_BRANCH).jar $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
 
 install:
        mkdir -p $(A2_NATIVE_INSTALL_TARGET)/$(A2_CATEGORY)
@@ -41,11 +48,36 @@ retrieve-llama3:
        git -C $(SRC_DIR) checkout $(LLAMA3_COMMIT)
        rm -rf $(SRC_DIR)/.git
 
+retrieve-jlama: SRC_DIR=src/jlama
+retrieve-jlama:
+       rm -rf $(SRC_DIR)
+       mkdir -p $(SRC_DIR)
+       git clone --branch v$(JLAMA_VERSION) https://github.com/tjake/Jlama.git $(SRC_DIR)
+       rm -rf $(SRC_DIR)/.git
+       rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/java/ com.github.tjake.jlama.native/src
+       # future: rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/java21/ com.github.tjake.jlama.native/src
+       rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/c/ $(JLAMA_JNI_SRC)
+
 BUNDLES = \
 de.kherud.llama \
 com.llama4j \
+com.github.tjake.jlama.native \
+
 
 clean:
        rm -rf $(BUILD_BASE)
 
+DEP_CATEGORIES = \
+log/syslogger/org.argeo.tp \
+org.argeo.tp.ml
+
 include $(SDK_SRC_BASE)/sdk/argeo-build/osgi.mk
+
+A2_NATIVE_CATEGORY=$(A2_OUTPUT)/lib/linux/$(shell uname -m)/$(A2_CATEGORY)
+
+jni-jlama:
+       mkdir -p $(A2_NATIVE_CATEGORY)
+       $(CC) -o $(A2_NATIVE_CATEGORY)/$(JLAMA_TARGET_EXEC) \
+        -O3 -mavx2 -march=native -Werror -Wno-attributes -fPIC -fno-omit-frame-pointer -Wunused-variable \
+        -shared -fPIC -fpic -Wl,-soname,$(JLAMA_TARGET_EXEC).$(JLAMA_BRANCH) \
+        -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JLAMA_JNI_SRC)/*.c

diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore
new file mode 100644 (file)
index 0000000..4f00cd9
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore
@@ -0,0 +1 @@
+/src/

diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore
new file mode 100644 (file)
index 0000000..68359a7
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore
@@ -0,0 +1,2 @@
+*.c
+*.h

diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF
new file mode 100644 (file)
index 0000000..67e54d5
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF
@@ -0,0 +1,17 @@
+Manifest-Version: 1.0\r
+Bundle-ManifestVersion: 2\r
+Export-Package: com.github.tjake.jlama.tensor.operations;uses:="com.gith\r
+ ub.tjake.jlama.safetensors,com.github.tjake.jlama.tensor",com.github.tj\r
+ ake.jlama.tensor.operations.cnative\r
+Import-Package: com.github.tjake.jlama.safetensors,com.github.tjake.jlam\r
+ a.tensor,com.github.tjake.jlama.tensor.operations.cnative,com.github.tj\r
+ ake.jlama.util,java.io,java.lang,java.lang.foreign,java.lang.invoke,jav\r
+ a.net,java.nio.file,java.nio.file.attribute,java.util,java.util.functio\r
+ n,org.slf4j\r
+Require-Capability: osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=20))"\r
+Bundle-RequiredExecutionEnvironment: JavaSE-20\r
+Bundle-Version: 0.2.1.next\r
+Automatic-Module-Name: com.github.tjake.jlama.native\r
+Bundle-SymbolicName: com.github.tjake.jlama.native\r
+Bundle-Name: com.github.tjake.jlama.native\r
+\r

diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd
new file mode 100644 (file)
index 0000000..7a6d19b
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd
@@ -0,0 +1,7 @@
+
+major: 0
+minor: 2
+micro: 1
+
+Require-Capability: osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=20))"
+Bundle-RequiredExecutionEnvironment: JavaSE-20
\ No newline at end of file

diff --git a/rebuild/org.argeo.tp.ml/jni/.gitignore b/rebuild/org.argeo.tp.ml/jni/.gitignore
new file mode 100644 (file)
index 0000000..1eb26b7
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/jni/.gitignore
@@ -0,0 +1,4 @@
+*.h
+*.c
+*.sh
+com_github_tjake_jlama

diff --git a/rebuild/org.argeo.tp.ml/src/.gitignore b/rebuild/org.argeo.tp.ml/src/.gitignore
index 2149a7e2dc3ad5f5a3bfc42ec20a3be0889b30de..674f05d6e2bdc7208830281d12eaa211f84395a4 100644 (file)
--- a/rebuild/org.argeo.tp.ml/src/.gitignore
+++ b/rebuild/org.argeo.tp.ml/src/.gitignore
@@ -1 +1,2 @@
 /jllama/
+/jlama/

diff --git a/repackage/Makefile b/repackage/Makefile
index b9c6bfe458db9ccf38d6f0703f909b359ee69bea..a95905b48c0b71a5573dbbb3a4aceffa3999fe31 100644 (file)
--- a/repackage/Makefile
+++ b/repackage/Makefile
@@ -36,6 +36,7 @@ org.argeo.tp.math \
 org.argeo.tp.jcr \
 org.argeo.tp.office \
 org.argeo.tp.earth \
+org.argeo.tp.ml \
 crypto/full/org.argeo.tp.crypto \
 
 install-for-minimal-cms:

diff --git a/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd b/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd
new file mode 100644 (file)
index 0000000..3f096a7
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd
@@ -0,0 +1,4 @@
+#FIXME: keep version in sync with sys!
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1
+Argeo-Origin-NoMetadataGeneration: true

diff --git a/repackage/org.argeo.tp.ml/com.google.re2j.bnd b/repackage/org.argeo.tp.ml/com.google.re2j.bnd
new file mode 100644 (file)
index 0000000..dba8739
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.google.re2j.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.google.re2j:re2j:1.7

diff --git a/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd b/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd
new file mode 100644 (file)
index 0000000..c545b5f
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd
@@ -0,0 +1,4 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.hubspot.jinjava:jinjava:2.7.2
+Import-Package:\
+*;resolution:="optional"

diff --git a/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd
new file mode 100644 (file)
index 0000000..b646d64
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd
@@ -0,0 +1,4 @@
+Argeo-Origin-M2: com.github.tjake:jlama-core
+Import-Package:\
+org.jctools.queues,\
+*

diff --git a/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled
new file mode 100644 (file)
index 0000000..0923e93
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled
@@ -0,0 +1 @@
+Argeo-Origin-M2: com.github.tjake:jlama-native

diff --git a/repackage/org.argeo.tp.ml/jlama/common.bnd b/repackage/org.argeo.tp.ml/jlama/common.bnd
new file mode 100644 (file)
index 0000000..0bc9b4a
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/common.bnd
@@ -0,0 +1,3 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :0.2.1
+Require-Capability: osgi.ee; filter="(&(osgi.ee=JavaSE)(version=20))"

diff --git a/repackage/org.argeo.tp.ml/langchain4j/common.bnd b/repackage/org.argeo.tp.ml/langchain4j/common.bnd
new file mode 100644 (file)
index 0000000..385e954
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/common.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :0.33.0

diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd
new file mode 100644 (file)
index 0000000..94dd7e8
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j

diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd
new file mode 100644 (file)
index 0000000..0be584d
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j-core

diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd
new file mode 100644 (file)
index 0000000..b7dc88c
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd
@@ -0,0 +1,2 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j-jlama
+Require-Capability: osgi.ee; filter="(&(osgi.ee=JavaSE)(version=20))"

diff --git a/repackage/org.argeo.tp.ml/opennlp/common.bnd b/repackage/org.argeo.tp.ml/opennlp/common.bnd
new file mode 100644 (file)
index 0000000..71435a1
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/opennlp/common.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :2.4.0

diff --git a/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd b/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd
new file mode 100644 (file)
index 0000000..266da35
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: org.apache.opennlp:opennlp-tools

diff --git a/repackage/org.argeo.tp.ml/org.jctools.bnd b/repackage/org.argeo.tp.ml/org.jctools.bnd
new file mode 100644 (file)
index 0000000..554b15b
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/org.jctools.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.jctools:jctools-core:4.0.5

diff --git a/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd b/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd
new file mode 100644 (file)
index 0000000..b657143
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.yaml:snakeyaml:2.3

diff --git a/repackage/org.argeo.tp.utils/com.google.code.gson.bnd b/repackage/org.argeo.tp.utils/com.google.code.gson.bnd
new file mode 100644 (file)
index 0000000..965ba34
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/com.google.code.gson.bnd
@@ -0,0 +1,3 @@
+SPDX-License-Identifier: Apache-2.0
+# Note: Used only by langchain
+Argeo-Origin-M2: com.google.code.gson:gson:2.11.0

diff --git a/repackage/org.argeo.tp.utils/com.google.common.bnd b/repackage/org.argeo.tp.utils/com.google.common.bnd
index 0686b7fd8258ee6744be5c61fdcce13ad407d4b8..2eac0662c2b41be8f60a0b6b5f6892f21d250bcb 100644 (file)
--- a/repackage/org.argeo.tp.utils/com.google.common.bnd
+++ b/repackage/org.argeo.tp.utils/com.google.common.bnd
@@ -1,5 +1,5 @@
 SPDX-License-Identifier: Apache-2.0
-# Note: Used only by h2gis
+# Note: Used only by h2gis and jlama
 Argeo-Origin-M2: com.google.guava:guava:33.2.1-jre
 Bundle-Version: 33.2.1.jre
 Import-Package:\

diff --git a/repackage/org.argeo.tp.utils/org.jsoup.bnd b/repackage/org.argeo.tp.utils/org.jsoup.bnd
new file mode 100644 (file)
index 0000000..7f5e8cd
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/org.jsoup.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+Argeo-Origin-M2: org.jsoup:jsoup:1.18.1

diff --git a/repackage/org.argeo.tp.utils/org.jspecify.bnd b/repackage/org.argeo.tp.utils/org.jspecify.bnd
new file mode 100644 (file)
index 0000000..2f74dfe
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/org.jspecify.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.jspecify:jspecify:1.0.0

diff --git a/sdk/output-argeo-tp-apps.target b/sdk/output-argeo-tp-apps.target
index 4347af16de11e5d02201c2871ee7bac244c195eb..6638e45c7a938871d891ecae4641ff3b08bfb89c 100644 (file)
--- a/sdk/output-argeo-tp-apps.target
+++ b/sdk/output-argeo-tp-apps.target
@@ -9,6 +9,8 @@
                <location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.earth" type="Directory"/>
                <location path="${project_loc:argeo-tp}/../output/a2/osgi/equinox/org.argeo.tp.eclipse" type="Directory"/>
                <location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.jcr" type="Directory"/>
+               <location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.ml" type="Directory"/>
+               <location path="${project_loc:argeo-tp}/../output/a2/lib/linux/x86_64/org.argeo.tp.ml" type="Directory"/>
                <location type="Target" uri="file:${project_loc:argeo-tp}/sdk/output-argeo-tp-platform.target"/>
        </locations>
 </target>
\ No newline at end of file