Skip to content

Commit

Permalink
Native libraries are searched in lib/polyglot project dir (#11874)
Browse files Browse the repository at this point in the history
* Update java.md docs

* Add docs

* Separate instance of HostClassLoader per project

* Insert default HostClassLoader for builtin lib

* Revert "Insert default HostClassLoader for builtin lib"

This reverts commit 84be175.

* Revert "Separate instance of HostClassLoader per project"

This reverts commit c98bdaf.

* Add NativeLibraryFinder

* OpenCV native lib is not copied to a temporary directory, but loaded from a dll

* std-image/compile does not copy opencv.jar

* Add JARUtils to sbt project

* Add *.dylib to gitignore

* Add error recovery to JARUtils

* std-image/compile extract native libraries from OpenCV.jar

* fmt

* JAR extraction uses sbt cache.

So that no extraction is done unnecessarily.

* Move extractNativeLibs task to StdBits

* NativeLibraryFinder strips version from os.arch

* fmt

* NativeLibFinder simplifies os name

* Extracted native lib dir conforms to the hierarchy naming convention

* Move NativeLibraryFinder to pkg and make it generic.

* Package has nativeLibraryDir method

* Fix typo in NativeLibraryFinder

* Fix path to nativeLibDir in pkg

* Fix build

* Add NativeLibraryFinder.listAllNativeLibraries

* Gather native libraries in EnsoLibraryFeature

* Stack of exception in EnsoLibraryFeature is printed to System.err.

The current printing is not visible on CI

* Update docs

* Update docs

* Add unit test for searching native lib

* Add printing system info to NativeLibraryFinderTest

* Always replace x86_64 by amd64

* Add list of supported names in docs

* Update changelog

* fmt docs

* EnsoLibraryFeature copies native libs next to the generated binary

* Flatten hierarchy of native libs in opencv

* native libs must have the correct platform-specific suffix

---------

Co-authored-by: Jaroslav Tulach <[email protected]>
  • Loading branch information
Akirathan and JaroslavTulach authored Jan 6, 2025
1 parent b759d75 commit 7658faf
Show file tree
Hide file tree
Showing 19 changed files with 577 additions and 60 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ bench-report*.xml
/enso.lib

*.dll
*.dylib
*.exe
*.pdb
*.so
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,15 @@
- A constructor or type definition with a single inline argument definition was
previously allowed to use spaces in the argument definition without
parentheses. [This is now a syntax error.][11856]
- [Native libraries of projects can be added to `polyglot/lib` directory][11874]
- [Redo stack is no longer lost when interacting with text literals][11908].
- Symetric, transitive and reflexive [equality for intersection types][11897]

[11777]: https://github.com/enso-org/enso/pull/11777
[11600]: https://github.com/enso-org/enso/pull/11600
[11856]: https://github.com/enso-org/enso/pull/11856
[11874]: https://github.com/enso-org/enso/pull/11874
[11908]: https://github.com/enso-org/enso/pull/11908
[11897]: https://github.com/enso-org/enso/pull/11897

# Enso 2024.5
Expand Down
26 changes: 19 additions & 7 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -3712,7 +3712,8 @@ lazy val `engine-runner` = project
// "-H:-DeleteLocalSymbols",
// you may need to set smallJdk := None to use following flags:
// "--trace-class-initialization=org.enso.syntax2.Parser",
"-Dnic=nic"
"-Dnic=nic",
"-Dorg.enso.feature.native.lib.output=" + (engineDistributionRoot.value / "bin")
),
mainClass = Some("org.enso.runner.Main"),
initializeAtRuntime = Seq(
Expand Down Expand Up @@ -4487,6 +4488,7 @@ def stdLibComponentRoot(name: String): File =
val `base-polyglot-root` = stdLibComponentRoot("Base") / "polyglot" / "java"
val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java"
val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java"
val `image-native-libs` = stdLibComponentRoot("Image") / "polyglot" / "lib"
val `google-api-polyglot-root` =
stdLibComponentRoot("Google_Api") / "polyglot" / "java"
val `database-polyglot-root` =
Expand Down Expand Up @@ -4654,6 +4656,10 @@ lazy val `std-table` = project
)
.dependsOn(`std-base` % "provided")

lazy val extractNativeLibs = taskKey[Unit](
"Helper task to extract native libraries from OpenCV JAR"
)

lazy val `std-image` = project
.in(file("std-bits") / "image")
.settings(
Expand All @@ -4669,15 +4675,21 @@ lazy val `std-image` = project
"org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided",
"org.openpnp" % "opencv" % opencvVersion
),
Compile / packageBin := Def.task {
val result = (Compile / packageBin).value
val _ = StdBits
.copyDependencies(
// Extract native libraries from opencv.jar, and put them under
// Standard/Image/polyglot/lib directory. The minimized opencv.jar will
// be put under Standard/Image/polyglot/java directory.
extractNativeLibs := {
StdBits
.extractNativeLibsFromOpenCV(
`image-polyglot-root`,
Seq("std-image.jar"),
ignoreScalaLibrary = true
`image-native-libs`,
opencvVersion
)
.value
},
Compile / packageBin := Def.task {
val result = (Compile / packageBin).value
val _ = extractNativeLibs.value
result
}.value
)
Expand Down
45 changes: 39 additions & 6 deletions docs/polyglot/java.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,27 +44,60 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects,
allowing Enso code to work with them through a runtime reflection-style
mechanism. It is comprised of the following components:

- `Java.lookup_class : Class.Path -> Maybe Class`: A function that lets users
look up a class by a given name on the runtime classpath.
- `Polyglot.instantiate : Class -> Object`: A function that lets users
instantiate a class into an object.
- `Java.lookup_class : Text -> Any`: A function that lets users look up a class
by a given name on the runtime classpath.
- A whole host of functions on the polyglot type that let you dynamically work
with object bindings.

An example can be found below:

```ruby
from Standard.Base.Polyglot import Java, polyglot

main =
class = Java.lookup_class "org.enso.example.TestClass"
instance = Polyglot.instantiate1 class (x -> x * 2)
instance = class.new (x -> x * 2)
method = Polyglot.get_member instance "callFunctionAndIncrement"
Polyglot.execute1 method 10
Polyglot.execute method 10
```

> The actionables for this section are:
>
> - Expand on the detail when there is time.
## Native libraries

Java can load native libraries using, e.g., the
[System.loadLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/System.html#loadLibrary(java.lang.String)>)
or
[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
methods. If a Java method loaded from the `polyglot/java` directory in project
`Proj` tries to load a native library via one of the aforementioned mechanisms,
the runtime system will look for the native library in the `polyglot/lib`
directory within the project `Proj`. The runtime system implements this by
overriding the
[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
method on the `ClassLoader` used to load the Java class.

The algorithm used to search for the native libraries within the `polyglot/lib`
directory hierarchy conforms to the
[NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni):
Lookup of library with name `native` works roughly in these steps:

- Add platform-specific prefix and/or suffix to the library name, e.g.,
`libnative.so` on Linux.
- Search for the library in the `polyglot/lib` directory.
- Search for the library in the `polyglot/lib/<arch>` directory, where `<arch>`
is the name of the architecture.
- Search for the library in the `polyglot/lib/<arch>/<os>` directory, where
`<os>` is the name of the operating system.

Supported names:

- Names for `<os>` are `linux`, `macos`, `windows`.
- Note that for simplicity we omit the versions of the operating systems.
- Names for architectures `<arch>` are `amd64`, `x86_64`, `x86_32`, `aarch64`.

## Download a Java Library from Maven Central

A typical use-case when bringing in some popular Java library into Enso
Expand Down
71 changes: 33 additions & 38 deletions engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,41 @@

import static scala.jdk.javaapi.CollectionConverters.asJava;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.LinkedHashSet;
import java.util.TreeSet;
import org.enso.compiler.core.EnsoParser;
import org.enso.compiler.core.ir.module.scope.imports.Polyglot;
import org.enso.filesystem.FileSystem$;
import org.enso.pkg.NativeLibraryFinder;
import org.enso.pkg.PackageManager$;
import org.graalvm.nativeimage.hosted.Feature;
import org.graalvm.nativeimage.hosted.RuntimeProxyCreation;
import org.graalvm.nativeimage.hosted.RuntimeReflection;
import org.graalvm.nativeimage.hosted.RuntimeResourceAccess;

public final class EnsoLibraryFeature implements Feature {
private static final String LIB_OUTPUT = "org.enso.feature.native.lib.output";
private final File nativeLibDir;

public EnsoLibraryFeature() {
var nativeLibOut = System.getProperty(LIB_OUTPUT);
if (nativeLibOut == null) {
throw new IllegalStateException("Missing system property: " + LIB_OUTPUT);
}
nativeLibDir = new File(nativeLibOut);
if (!nativeLibDir.exists() || !nativeLibDir.isDirectory()) {
var created = nativeLibDir.mkdirs();
if (!created) {
throw new IllegalStateException("Cannot create directory: " + nativeLibDir);
}
}
}

@Override
public void beforeAnalysis(BeforeAnalysisAccess access) {
try {
registerOpenCV(access.getApplicationClassLoader());
} catch (ReflectiveOperationException ex) {
ex.printStackTrace();
throw new IllegalStateException(ex);
}

var libs = new LinkedHashSet<Path>();
for (var p : access.getApplicationClassPath()) {
var p1 = p.getParent();
Expand Down Expand Up @@ -53,6 +67,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
*/

var classes = new TreeSet<String>();
var nativeLibPaths = new TreeSet<String>();
try {
for (var p : libs) {
var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile());
Expand Down Expand Up @@ -90,46 +105,26 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
}
}
}
if (pkg.nativeLibraryDir().exists()) {
var nativeLibs =
NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs());
for (var nativeLib : nativeLibs) {
var out = new File(nativeLibDir, nativeLib.getName());
Files.copy(nativeLib.toPath(), out.toPath());
nativeLibPaths.add(out.getAbsolutePath());
}
}
}
}
} catch (Exception ex) {
ex.printStackTrace();
ex.printStackTrace(System.err);
throw new IllegalStateException(ex);
}
System.err.println("Summary for polyglot import java:");
for (var className : classes) {
System.err.println(" " + className);
}
System.err.println("Registered " + classes.size() + " classes for reflection");
}

private static void registerOpenCV(ClassLoader cl) throws ReflectiveOperationException {
var moduleOpenCV = cl.getUnnamedModule();
var currentOS = System.getProperty("os.name").toUpperCase().replaceAll(" .*$", "");

var libOpenCV =
switch (currentOS) {
case "LINUX" -> "nu/pattern/opencv/linux/x86_64/libopencv_java470.so";
case "WINDOWS" -> "nu/pattern/opencv/windows/x86_64/opencv_java470.dll";
case "MAC" -> {
var arch = System.getProperty("os.arch").toUpperCase();
yield switch (arch) {
case "X86_64" -> "nu/pattern/opencv/osx/x86_64/libopencv_java470.dylib";
case "AARCH64" -> "nu/pattern/opencv/osx/ARMv8/libopencv_java470.dylib";
default -> null;
};
}
default -> null;
};

if (libOpenCV != null) {
var verify = cl.getResource(libOpenCV);
if (verify == null) {
throw new IllegalStateException("Cannot find " + libOpenCV + " resource in " + cl);
}
RuntimeResourceAccess.addResource(moduleOpenCV, libOpenCV);
} else {
throw new IllegalStateException("No resource suggested for " + currentOS);
}
System.err.println("Copied native libraries: " + nativeLibPaths + " into " + nativeLibDir);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public class SerializationManagerTest {

@Before
public void setup() {
packageManager = new PackageManager<>(new TruffleFileSystem());
packageManager = new PackageManager<>(TruffleFileSystem.INSTANCE);
interpreterContext = new InterpreterContext(x -> x);
ensoContext =
interpreterContext
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package org.enso.interpreter.test;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;

import com.oracle.truffle.api.TruffleFile;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.List;
import org.enso.editions.LibraryName;
import org.enso.interpreter.runtime.util.TruffleFileSystem;
import org.enso.pkg.NativeLibraryFinder;
import org.enso.pkg.Package;
import org.enso.test.utils.ContextUtils;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;

public class NativeLibraryFinderTest {

@Rule public final TestRule printContextRule = new PrintSystemInfoRule();
private Package<TruffleFile> stdImgPkg;

@Test
public void standardImageShouldHaveNativeLib() {
try (var ctx = ContextUtils.createDefaultContext()) {
// Evaluate dummy sources to force loading Standard.Image
ContextUtils.evalModule(
ctx, """
from Standard.Image import all
main = 42
""");
var ensoCtx = ContextUtils.leakContext(ctx);
var stdImg =
ensoCtx
.getPackageRepository()
.getPackageForLibraryJava(LibraryName.apply("Standard", "Image"));
assertThat(stdImg.isPresent(), is(true));
this.stdImgPkg = stdImg.get();
var nativeLibs =
NativeLibraryFinder.listAllNativeLibraries(stdImg.get(), TruffleFileSystem.INSTANCE);
assertThat(
"There should be just single native lib in Standard.Image", nativeLibs.size(), is(1));
}
}

public final class PrintSystemInfoRule implements TestRule {
@Override
public Statement apply(Statement base, Description description) {
return new Statement() {
@Override
public void evaluate() {
try {
base.evaluate();
} catch (Throwable e) {
var sb = new StringBuilder();
sb.append(System.lineSeparator());
sb.append(" os.name: ")
.append(System.getProperty("os.name"))
.append(System.lineSeparator());
sb.append(" os.arch: ")
.append(System.getProperty("os.arch"))
.append(System.lineSeparator());
var mappedLibName = System.mapLibraryName("opencv_java470");
sb.append(" Mapped library name: ")
.append(mappedLibName)
.append(System.lineSeparator());
if (stdImgPkg != null) {
sb.append(" Contents of Standard.Image native library dir:")
.append(System.lineSeparator());
var nativeLibDir = stdImgPkg.nativeLibraryDir();
var nativeLibPath = Path.of(nativeLibDir.getAbsoluteFile().getPath());
var contents = contentsOfDir(nativeLibPath);
contents.forEach(
path -> sb.append(" ").append(path).append(System.lineSeparator()));
}
throw new AssertionError(sb.toString(), e);
}
}
};
}
}

private static List<String> contentsOfDir(Path dir) {
var contents = new ArrayList<String>();
var fileVisitor =
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
contents.add(file.toAbsolutePath().toString());
return FileVisitResult.CONTINUE;
}
};
try {
Files.walkFileTree(dir, fileVisitor);
} catch (IOException e) {
throw new AssertionError(e);
}
return contents;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ public EnsoContext(

/** Perform expensive initialization logic for the context. */
public void initialize() {
TruffleFileSystem fs = new TruffleFileSystem();
TruffleFileSystem fs = TruffleFileSystem.INSTANCE;
PackageManager<TruffleFile> packageManager = new PackageManager<>(fs);

Optional<TruffleFile> projectRoot = OptionsHelper.getProjectRoot(environment);
Expand Down
Loading

0 comments on commit 7658faf

Please sign in to comment.