From a7df3506760a16609eaf350afeaa60bd073c7941 Mon Sep 17 00:00:00 2001
From: Mathieu Baudier <mbaudier@argeo.org>
Date: Sun, 1 Sep 2024 13:06:21 +0200
Subject: [PATCH] Java components for machine learning

---
 rebuild/Makefile                              |  5 +++
 rebuild/org.argeo.tp.ml/Makefile              | 34 ++++++++++++++++++-
 .../com.github.tjake.jlama.native/.gitignore  |  1 +
 .../META-INF/.gitignore                       |  2 ++
 .../META-INF/MANIFEST.MF                      | 17 ++++++++++
 .../com.github.tjake.jlama.native/bnd.bnd     |  7 ++++
 rebuild/org.argeo.tp.ml/jni/.gitignore        |  4 +++
 rebuild/org.argeo.tp.ml/src/.gitignore        |  1 +
 repackage/Makefile                            |  1 +
 ...l.jackson.core.jackson.dataformat.yaml.bnd |  4 +++
 repackage/org.argeo.tp.ml/com.google.re2j.bnd |  2 ++
 .../org.argeo.tp.ml/com.hubspot.jinjava.bnd   |  4 +++
 .../jlama/com.github.tjake.jlama.bnd          |  4 +++
 ...com.github.tjake.jlama.native.bnd.disabled |  1 +
 repackage/org.argeo.tp.ml/jlama/common.bnd    |  3 ++
 .../org.argeo.tp.ml/langchain4j/common.bnd    |  2 ++
 .../langchain4j/dev.langchain4j.bnd           |  1 +
 .../langchain4j/dev.langchain4j.core.bnd      |  1 +
 .../langchain4j/dev.langchain4j.jlama.bnd     |  2 ++
 repackage/org.argeo.tp.ml/opennlp/common.bnd  |  2 ++
 .../opennlp/org.apache.opennlp.tools.bnd      |  1 +
 repackage/org.argeo.tp.ml/org.jctools.bnd     |  2 ++
 .../org.argeo.tp.ml/org.yaml.snakeyaml.bnd    |  2 ++
 .../com.google.code.gson.bnd                  |  3 ++
 .../org.argeo.tp.utils/com.google.common.bnd  |  2 +-
 repackage/org.argeo.tp.utils/org.jsoup.bnd    |  2 ++
 repackage/org.argeo.tp.utils/org.jspecify.bnd |  2 ++
 sdk/output-argeo-tp-apps.target               |  2 ++
 28 files changed, 112 insertions(+), 2 deletions(-)
 create mode 100644 rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore
 create mode 100644 rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore
 create mode 100644 rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF
 create mode 100644 rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd
 create mode 100644 rebuild/org.argeo.tp.ml/jni/.gitignore
 create mode 100644 repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd
 create mode 100644 repackage/org.argeo.tp.ml/com.google.re2j.bnd
 create mode 100644 repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd
 create mode 100644 repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd
 create mode 100644 repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled
 create mode 100644 repackage/org.argeo.tp.ml/jlama/common.bnd
 create mode 100644 repackage/org.argeo.tp.ml/langchain4j/common.bnd
 create mode 100644 repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd
 create mode 100644 repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd
 create mode 100644 repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd
 create mode 100644 repackage/org.argeo.tp.ml/opennlp/common.bnd
 create mode 100644 repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd
 create mode 100644 repackage/org.argeo.tp.ml/org.jctools.bnd
 create mode 100644 repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd
 create mode 100644 repackage/org.argeo.tp.utils/com.google.code.gson.bnd
 create mode 100644 repackage/org.argeo.tp.utils/org.jsoup.bnd
 create mode 100644 repackage/org.argeo.tp.utils/org.jspecify.bnd

diff --git a/rebuild/Makefile b/rebuild/Makefile
index 243c3e0..9b43136 100644
--- a/rebuild/Makefile
+++ b/rebuild/Makefile
@@ -2,12 +2,17 @@
 
 all:
 	make -C org.argeo.tp.sys
+	make -C org.argeo.tp.ml
+
 install:
 	make -C org.argeo.tp.sys install
+	make -C org.argeo.tp.ml install
 	
 uninstall:
 	make -C org.argeo.tp.sys uninstall
+	make -C org.argeo.tp.ml uninstall
 	
 clean:
 	make -C org.argeo.tp.sys clean
+	make -C org.argeo.tp.ml clean
 	
\ No newline at end of file
diff --git a/rebuild/org.argeo.tp.ml/Makefile b/rebuild/org.argeo.tp.ml/Makefile
index 41203c5..6e06256 100644
--- a/rebuild/org.argeo.tp.ml/Makefile
+++ b/rebuild/org.argeo.tp.ml/Makefile
@@ -5,6 +5,9 @@ export NO_SDK_LEGAL := true
 
 A2_CATEGORY = org.argeo.tp.ml
 
+JLAMA_JNI_SRC=jni/com_github_tjake_jlama
+JLAMA_TARGET_EXEC=libjlama.so
+
 ## FIXME - DON'T FORGET TO UPDATE THE VERSION IN THE RELATED bnd.bnd FILE!
 JLLAMA_BRANCH=3.3
 JLLAMA_VERSION=$(JLLAMA_BRANCH).0
@@ -12,9 +15,13 @@ JLLAMA_VERSION=$(JLLAMA_BRANCH).0
 LLAMA3_COMMIT=5f602500beb593dce2726df7e2dd08b0803aa86b
 LLAMA3_VERSION=0.1.0
 
-all: retrieve-jllama retrieve-llama3 osgi
+JLAMA_BRANCH=0.2
+JLAMA_VERSION=$(JLAMA_BRANCH).1
+
+all: retrieve-jllama retrieve-llama3 retrieve-jlama osgi jni-jlama
 	mkdir -p $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
 	mv $(SDK_BUILD_BASE)/a2/$(A2_CATEGORY)/de.kherud.llama.$(JLLAMA_BRANCH).jar $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
+	mv $(SDK_BUILD_BASE)/a2/$(A2_CATEGORY)/com.github.tjake.jlama.native.$(JLAMA_BRANCH).jar $(SDK_BUILD_BASE)/a2/$(TARGET_ARCH_CATEGORY_PREFIX)/$(A2_CATEGORY)
 
 install:
 	mkdir -p $(A2_NATIVE_INSTALL_TARGET)/$(A2_CATEGORY)
@@ -41,11 +48,36 @@ retrieve-llama3:
 	git -C $(SRC_DIR) checkout $(LLAMA3_COMMIT)
 	rm -rf $(SRC_DIR)/.git
 
+retrieve-jlama: SRC_DIR=src/jlama
+retrieve-jlama:
+	rm -rf $(SRC_DIR)
+	mkdir -p $(SRC_DIR)
+	git clone --branch v$(JLAMA_VERSION) https://github.com/tjake/Jlama.git $(SRC_DIR)
+	rm -rf $(SRC_DIR)/.git
+	rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/java/ com.github.tjake.jlama.native/src
+	# future: rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/java21/ com.github.tjake.jlama.native/src
+	rsync -a --delete --exclude module-info.java $(SRC_DIR)/jlama-native/src/main/c/ $(JLAMA_JNI_SRC)
+
 BUNDLES = \
 de.kherud.llama \
 com.llama4j \
+com.github.tjake.jlama.native \
+
 
 clean:
 	rm -rf $(BUILD_BASE)
 
+DEP_CATEGORIES = \
+log/syslogger/org.argeo.tp \
+org.argeo.tp.ml
+
 include $(SDK_SRC_BASE)/sdk/argeo-build/osgi.mk
+
+A2_NATIVE_CATEGORY=$(A2_OUTPUT)/lib/linux/$(shell uname -m)/$(A2_CATEGORY)
+
+jni-jlama:
+	mkdir -p $(A2_NATIVE_CATEGORY)
+	$(CC) -o $(A2_NATIVE_CATEGORY)/$(JLAMA_TARGET_EXEC) \
+	 -O3 -mavx2 -march=native -Werror -Wno-attributes -fPIC -fno-omit-frame-pointer -Wunused-variable \
+	 -shared -fPIC -fpic -Wl,-soname,$(JLAMA_TARGET_EXEC).$(JLAMA_BRANCH) \
+	 -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/linux $(JLAMA_JNI_SRC)/*.c
diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore
new file mode 100644
index 0000000..4f00cd9
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/.gitignore
@@ -0,0 +1 @@
+/src/
diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore
new file mode 100644
index 0000000..68359a7
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/.gitignore
@@ -0,0 +1,2 @@
+*.c
+*.h
diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF
new file mode 100644
index 0000000..67e54d5
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/META-INF/MANIFEST.MF
@@ -0,0 +1,17 @@
+Manifest-Version: 1.0
+Bundle-ManifestVersion: 2
+Export-Package: com.github.tjake.jlama.tensor.operations;uses:="com.gith
+ ub.tjake.jlama.safetensors,com.github.tjake.jlama.tensor",com.github.tj
+ ake.jlama.tensor.operations.cnative
+Import-Package: com.github.tjake.jlama.safetensors,com.github.tjake.jlam
+ a.tensor,com.github.tjake.jlama.tensor.operations.cnative,com.github.tj
+ ake.jlama.util,java.io,java.lang,java.lang.foreign,java.lang.invoke,jav
+ a.net,java.nio.file,java.nio.file.attribute,java.util,java.util.functio
+ n,org.slf4j
+Require-Capability: osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=20))"
+Bundle-RequiredExecutionEnvironment: JavaSE-20
+Bundle-Version: 0.2.1.next
+Automatic-Module-Name: com.github.tjake.jlama.native
+Bundle-SymbolicName: com.github.tjake.jlama.native
+Bundle-Name: com.github.tjake.jlama.native
+
diff --git a/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd
new file mode 100644
index 0000000..7a6d19b
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/com.github.tjake.jlama.native/bnd.bnd
@@ -0,0 +1,7 @@
+
+major: 0
+minor: 2
+micro: 1
+
+Require-Capability: osgi.ee;filter:="(&(osgi.ee=JavaSE)(version=20))"
+Bundle-RequiredExecutionEnvironment: JavaSE-20
\ No newline at end of file
diff --git a/rebuild/org.argeo.tp.ml/jni/.gitignore b/rebuild/org.argeo.tp.ml/jni/.gitignore
new file mode 100644
index 0000000..1eb26b7
--- /dev/null
+++ b/rebuild/org.argeo.tp.ml/jni/.gitignore
@@ -0,0 +1,4 @@
+*.h
+*.c
+*.sh
+com_github_tjake_jlama
diff --git a/rebuild/org.argeo.tp.ml/src/.gitignore b/rebuild/org.argeo.tp.ml/src/.gitignore
index 2149a7e..674f05d 100644
--- a/rebuild/org.argeo.tp.ml/src/.gitignore
+++ b/rebuild/org.argeo.tp.ml/src/.gitignore
@@ -1 +1,2 @@
 /jllama/
+/jlama/
diff --git a/repackage/Makefile b/repackage/Makefile
index b9c6bfe..a95905b 100644
--- a/repackage/Makefile
+++ b/repackage/Makefile
@@ -36,6 +36,7 @@ org.argeo.tp.math \
 org.argeo.tp.jcr \
 org.argeo.tp.office \
 org.argeo.tp.earth \
+org.argeo.tp.ml \
 crypto/full/org.argeo.tp.crypto \
 
 install-for-minimal-cms:
diff --git a/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd b/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd
new file mode 100644
index 0000000..3f096a7
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.fasterxml.jackson.core.jackson.dataformat.yaml.bnd
@@ -0,0 +1,4 @@
+#FIXME: keep version in sync with sys!
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.17.1
+Argeo-Origin-NoMetadataGeneration: true
diff --git a/repackage/org.argeo.tp.ml/com.google.re2j.bnd b/repackage/org.argeo.tp.ml/com.google.re2j.bnd
new file mode 100644
index 0000000..dba8739
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.google.re2j.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.google.re2j:re2j:1.7
diff --git a/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd b/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd
new file mode 100644
index 0000000..c545b5f
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/com.hubspot.jinjava.bnd
@@ -0,0 +1,4 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: com.hubspot.jinjava:jinjava:2.7.2
+Import-Package:\
+*;resolution:="optional"
diff --git a/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd
new file mode 100644
index 0000000..b646d64
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.bnd
@@ -0,0 +1,4 @@
+Argeo-Origin-M2: com.github.tjake:jlama-core
+Import-Package:\
+org.jctools.queues,\
+*
diff --git a/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled
new file mode 100644
index 0000000..0923e93
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/com.github.tjake.jlama.native.bnd.disabled
@@ -0,0 +1 @@
+Argeo-Origin-M2: com.github.tjake:jlama-native
diff --git a/repackage/org.argeo.tp.ml/jlama/common.bnd b/repackage/org.argeo.tp.ml/jlama/common.bnd
new file mode 100644
index 0000000..0bc9b4a
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/jlama/common.bnd
@@ -0,0 +1,3 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :0.2.1
+Require-Capability: osgi.ee; filter="(&(osgi.ee=JavaSE)(version=20))"
diff --git a/repackage/org.argeo.tp.ml/langchain4j/common.bnd b/repackage/org.argeo.tp.ml/langchain4j/common.bnd
new file mode 100644
index 0000000..385e954
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/common.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :0.33.0
diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd
new file mode 100644
index 0000000..94dd7e8
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j
diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd
new file mode 100644
index 0000000..0be584d
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.core.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j-core
diff --git a/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd
new file mode 100644
index 0000000..b7dc88c
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/langchain4j/dev.langchain4j.jlama.bnd
@@ -0,0 +1,2 @@
+Argeo-Origin-M2: dev.langchain4j:langchain4j-jlama
+Require-Capability: osgi.ee; filter="(&(osgi.ee=JavaSE)(version=20))"
diff --git a/repackage/org.argeo.tp.ml/opennlp/common.bnd b/repackage/org.argeo.tp.ml/opennlp/common.bnd
new file mode 100644
index 0000000..71435a1
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/opennlp/common.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: :2.4.0
diff --git a/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd b/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd
new file mode 100644
index 0000000..266da35
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/opennlp/org.apache.opennlp.tools.bnd
@@ -0,0 +1 @@
+Argeo-Origin-M2: org.apache.opennlp:opennlp-tools
diff --git a/repackage/org.argeo.tp.ml/org.jctools.bnd b/repackage/org.argeo.tp.ml/org.jctools.bnd
new file mode 100644
index 0000000..554b15b
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/org.jctools.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.jctools:jctools-core:4.0.5
diff --git a/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd b/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd
new file mode 100644
index 0000000..b657143
--- /dev/null
+++ b/repackage/org.argeo.tp.ml/org.yaml.snakeyaml.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.yaml:snakeyaml:2.3
diff --git a/repackage/org.argeo.tp.utils/com.google.code.gson.bnd b/repackage/org.argeo.tp.utils/com.google.code.gson.bnd
new file mode 100644
index 0000000..965ba34
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/com.google.code.gson.bnd
@@ -0,0 +1,3 @@
+SPDX-License-Identifier: Apache-2.0
+# Note: Used only by langchain
+Argeo-Origin-M2: com.google.code.gson:gson:2.11.0
diff --git a/repackage/org.argeo.tp.utils/com.google.common.bnd b/repackage/org.argeo.tp.utils/com.google.common.bnd
index 0686b7f..2eac066 100644
--- a/repackage/org.argeo.tp.utils/com.google.common.bnd
+++ b/repackage/org.argeo.tp.utils/com.google.common.bnd
@@ -1,5 +1,5 @@
 SPDX-License-Identifier: Apache-2.0
-# Note: Used only by h2gis
+# Note: Used only by h2gis and jlama
 Argeo-Origin-M2: com.google.guava:guava:33.2.1-jre
 Bundle-Version: 33.2.1.jre
 Import-Package:\
diff --git a/repackage/org.argeo.tp.utils/org.jsoup.bnd b/repackage/org.argeo.tp.utils/org.jsoup.bnd
new file mode 100644
index 0000000..7f5e8cd
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/org.jsoup.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: MIT
+Argeo-Origin-M2: org.jsoup:jsoup:1.18.1
diff --git a/repackage/org.argeo.tp.utils/org.jspecify.bnd b/repackage/org.argeo.tp.utils/org.jspecify.bnd
new file mode 100644
index 0000000..2f74dfe
--- /dev/null
+++ b/repackage/org.argeo.tp.utils/org.jspecify.bnd
@@ -0,0 +1,2 @@
+SPDX-License-Identifier: Apache-2.0
+Argeo-Origin-M2: org.jspecify:jspecify:1.0.0
diff --git a/sdk/output-argeo-tp-apps.target b/sdk/output-argeo-tp-apps.target
index 4347af1..6638e45 100644
--- a/sdk/output-argeo-tp-apps.target
+++ b/sdk/output-argeo-tp-apps.target
@@ -9,6 +9,8 @@
 		<location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.earth" type="Directory"/>
 		<location path="${project_loc:argeo-tp}/../output/a2/osgi/equinox/org.argeo.tp.eclipse" type="Directory"/>
 		<location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.jcr" type="Directory"/>
+		<location path="${project_loc:argeo-tp}/../output/a2/org.argeo.tp.ml" type="Directory"/>
+		<location path="${project_loc:argeo-tp}/../output/a2/lib/linux/x86_64/org.argeo.tp.ml" type="Directory"/>
 		<location type="Target" uri="file:${project_loc:argeo-tp}/sdk/output-argeo-tp-platform.target"/>
 	</locations>
 </target>
\ No newline at end of file
-- 
2.39.5