Skip to content

Commit

Permalink
Avoid using shared resource streams between class loaders when extrac…
Browse files Browse the repository at this point in the history
…ting the native library. (#578)

This changes makes it so that we replace Class#getResoirceAsStream() with a custom method that finds the resources and creates a stream but disables caching the stream. This is necessary in environments with where the native library has to be extracted and used with multiple class loaders that get closed.

Specifically, I was observing the following [JDK-8205976](https://bugs.openjdk.java.net/browse/JDK-8205976) in a [Gradle](https://gradle.org/) project with concurrent tasks that use sqlite-jdbc. The exact exception I was observing was:
```
java.io.IOException: Stream closed
        at java.base/java.util.zip.InflaterInputStream.ensureOpen(InflaterInputStream.java:68)
        at java.base/java.util.zip.InflaterInputStream.read(InflaterInputStream.java:143)
        at java.base/java.io.FilterInputStream.read(FilterInputStream.java:133)
        at java.base/java.io.BufferedInputStream.fill(BufferedInputStream.java:252)
        at java.base/java.io.BufferedInputStream.read(BufferedInputStream.java:271)
        at org.sqlite.SQLiteJDBCLoader.contentsEquals(SQLiteJDBCLoader.java:179)
        at org.sqlite.SQLiteJDBCLoader.extractAndLoadLibraryFile(SQLiteJDBCLoader.java:243)
        at org.sqlite.SQLiteJDBCLoader.loadSQLiteNativeLibrary(SQLiteJDBCLoader.java:344)
        at org.sqlite.SQLiteJDBCLoader.initialize(SQLiteJDBCLoader.java:67)
 ```
 Which lead me to JDK-8205976 after lots of logging. The test is a big complicated but sets ups the exact case for when this occurs. Multiple concurrent threads with isolated class loaders initializing the library for which the resource on all point to the same jar.
  • Loading branch information
danysantiago authored Jun 27, 2021
1 parent 2eeb817 commit bf5e840
Show file tree
Hide file tree
Showing 2 changed files with 138 additions and 3 deletions.
28 changes: 25 additions & 3 deletions src/main/java/org/sqlite/SQLiteJDBCLoader.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
Expand Down Expand Up @@ -204,7 +205,7 @@ private static boolean extractAndLoadLibraryFile(String libFolderForCurrentOS, S

try {
// Extract a native library file into the target directory
InputStream reader = SQLiteJDBCLoader.class.getResourceAsStream(nativeLibraryFilePath);
InputStream reader = getResourceAsStream(nativeLibraryFilePath);
if(!extractedLckFile.exists()) {
new FileOutputStream(extractedLckFile).close();
}
Expand Down Expand Up @@ -237,7 +238,7 @@ private static boolean extractAndLoadLibraryFile(String libFolderForCurrentOS, S

// Check whether the contents are properly copied from the resource folder
{
InputStream nativeIn = SQLiteJDBCLoader.class.getResourceAsStream(nativeLibraryFilePath);
InputStream nativeIn = getResourceAsStream(nativeLibraryFilePath);
InputStream extractedLibIn = new FileInputStream(extractedLibFile);
try {
if(!contentsEquals(nativeIn, extractedLibIn)) {
Expand All @@ -256,12 +257,33 @@ private static boolean extractAndLoadLibraryFile(String libFolderForCurrentOS, S
return loadNativeLibrary(targetFolder, extractedLibFileName);
}
catch(IOException e) {
System.err.println(e.getMessage());
e.printStackTrace();
return false;
}

}

// Replacement of java.lang.Class#getResourceAsStream(String) to disable sharing the resource stream
// in multiple class loaders and specifically to avoid https://bugs.openjdk.java.net/browse/JDK-8205976
private static InputStream getResourceAsStream(String name) {
// Remove leading '/' since all our resource paths include a leading directory
// See: /~https://github.com/openjdk/jdk/blob/master/src/java.base/share/classes/java/lang/Class.java#L3054
String resolvedName = name.substring(1);
ClassLoader cl = SQLiteJDBCLoader.class.getClassLoader();
URL url = cl.getResource(resolvedName);
if (url == null) {
return null;
}
try {
URLConnection connection = url.openConnection();
connection.setDefaultUseCaches(false);
return connection.getInputStream();
} catch (IOException e) {
e.printStackTrace();
return null;
}
}

/**
* Loads native library using the given path and name of the library.
*
Expand Down
113 changes: 113 additions & 0 deletions src/test/java/org/sqlite/SQLiteJDBCLoaderTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,14 @@
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.lang.reflect.Method;
import java.net.URL;
import java.net.URLClassLoader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
Expand All @@ -37,6 +45,8 @@
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;

import org.junit.After;
import org.junit.Assert;
Expand Down Expand Up @@ -146,4 +156,107 @@ public void run() {
pool.awaitTermination(3, TimeUnit.SECONDS);
assertEquals(32, completedThreads.get());
}

@Test
public void multipleClassLoader() throws Throwable {
// Get current classpath
String[] stringUrls = System.getProperty("java.class.path")
.split(System.getProperty("path.separator"));
// Find the classes under test.
String targetFolderName = "sqlite-jdbc/target/classes";
File classesDir = null;
String classesDirPrefix = null;
for (String stringUrl : stringUrls) {
int indexOf = stringUrl.indexOf(targetFolderName);
if (indexOf != -1) {
classesDir = new File(stringUrl);
classesDirPrefix = stringUrl.substring(0, indexOf + targetFolderName.length());
break;
}
}
if (classesDir == null) {
Assert.fail("Couldn't find classes under test.");
}
// Create a JAR file out the classes and resources
File jarFile = File.createTempFile("jar-for-test-", ".jar");
createJar(classesDir, classesDirPrefix, jarFile);
URL[] jarUrl = new URL[] { new URL("file://" + jarFile.getAbsolutePath()) };

final AtomicInteger completedThreads = new AtomicInteger(0);
ExecutorService pool = Executors.newFixedThreadPool(4);
for (int i = 0; i < 4; i++) {
final int sleepMillis = i;
pool.execute(new Runnable() {
public void run() {
try {
Thread.sleep(sleepMillis * 10);
// Create an isolated class loader, it should load *different* instances
// of SQLiteJDBCLoader.class
URLClassLoader classLoader = new URLClassLoader(
jarUrl, ClassLoader.getSystemClassLoader().getParent());
Class<?> clazz =
classLoader.loadClass("org.sqlite.SQLiteJDBCLoader");
Method initMethod = clazz.getDeclaredMethod("initialize");
initMethod.invoke(null);
classLoader.close();
} catch (Throwable e) {
e.printStackTrace();
Assert.fail(e.getLocalizedMessage());
}
completedThreads.incrementAndGet();
}
});
}
pool.shutdown();
pool.awaitTermination(3, TimeUnit.SECONDS);
assertEquals(4, completedThreads.get());
}

private static void createJar(File inputDir, String changeDir, File outputFile) throws IOException {
JarOutputStream target = new JarOutputStream(new FileOutputStream(outputFile));
addJarEntry(inputDir, changeDir, target);
target.close();
}

private static void addJarEntry(File source, String changeDir, JarOutputStream target) throws IOException {
BufferedInputStream in = null;
try {
if (source.isDirectory()) {
String name = source.getPath().replace("\\", "/");
if (!name.isEmpty()) {
if (!name.endsWith("/")) {
name += "/";
}
JarEntry entry = new JarEntry(name.substring(changeDir.length() + 1));
entry.setTime(source.lastModified());
target.putNextEntry(entry);
target.closeEntry();
}
for (File nestedFile : source.listFiles()) {
addJarEntry(nestedFile, changeDir, target);
}
return;
}

JarEntry entry = new JarEntry(
source.getPath().replace("\\", "/").substring(changeDir.length() + 1));
entry.setTime(source.lastModified());
target.putNextEntry(entry);
in = new BufferedInputStream(new FileInputStream(source));

byte[] buffer = new byte[8192];
while (true) {
int count = in.read(buffer);
if (count == -1) {
break;
}
target.write(buffer, 0, count);
}
target.closeEntry();
} finally {
if (in != null) {
in.close();
}
}
}
}

0 comments on commit bf5e840

Please sign in to comment.