Download dependencies in parallel

This commit is contained in:
Luck 2020-05-06 02:50:19 +01:00
parent f6460c2802
commit fe32aa2d33
No known key found for this signature in database
GPG Key ID: EFA9B3EC5FD90F8B
5 changed files with 370 additions and 227 deletions

View File

@ -30,11 +30,15 @@ import com.google.common.collect.ImmutableList;
import me.lucko.luckperms.common.dependencies.relocation.Relocation;
import me.lucko.luckperms.common.dependencies.relocation.RelocationHelper;
import java.net.MalformedURLException;
import java.net.URL;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Base64;
import java.util.List;
/**
* The dependencies used by LuckPerms.
*/
public enum Dependency {
ASM(
@ -281,13 +285,11 @@ public enum Dependency {
Relocation.of("toml4j", "com{}moandjiezana{}toml")
);
private final List<URL> urls;
private final String mavenRepoPath;
private final String version;
private final byte[] checksum;
private final List<Relocation> relocations;
private static final String MAVEN_CENTRAL_REPO = "https://repo1.maven.org/maven2/";
private static final String LUCK_MIRROR_REPO = "https://nexus.lucko.me/repository/maven-central/";
private static final String MAVEN_FORMAT = "%s/%s/%s/%s-%s.jar";
Dependency(String groupId, String artifactId, String version, String checksum) {
@ -295,21 +297,13 @@ public enum Dependency {
}
Dependency(String groupId, String artifactId, String version, String checksum, Relocation... relocations) {
String path = String.format(MAVEN_FORMAT,
this.mavenRepoPath = String.format(MAVEN_FORMAT,
rewriteEscaping(groupId).replace(".", "/"),
rewriteEscaping(artifactId),
version,
rewriteEscaping(artifactId),
version
);
try {
this.urls = ImmutableList.of(
new URL(LUCK_MIRROR_REPO + path),
new URL(MAVEN_CENTRAL_REPO + path)
);
} catch (MalformedURLException e) {
throw new RuntimeException(e); // propagate
}
this.version = version;
this.checksum = Base64.getDecoder().decode(checksum);
this.relocations = ImmutableList.copyOf(relocations);
@ -319,53 +313,71 @@ public enum Dependency {
return s.replace("{}", ".");
}
/*
public static void main(String[] args) throws Exception {
java.security.MessageDigest digest = java.security.MessageDigest.getInstance("SHA-256");
for (Dependency dependency : values()) {
List<byte[]> hashes = new java.util.ArrayList<>();
for (URL url : dependency.getUrls()) {
java.net.URLConnection connection = url.openConnection();
connection.setRequestProperty("User-Agent", "luckperms");
try (java.io.InputStream in = connection.getInputStream()) {
byte[] bytes = com.google.common.io.ByteStreams.toByteArray(in);
if (bytes.length == 0) {
throw new RuntimeException("Empty stream");
}
hashes.add(digest.digest(bytes));
}
}
for (int i = 0; i < hashes.size(); i++) {
byte[] hash = hashes.get(i);
if (!java.util.Arrays.equals(hash, dependency.getChecksum())) {
System.out.println("NO MATCH - REPO " + i + " - " + dependency.name() + ": " + Base64.getEncoder().encodeToString(hash));
}
}
}
}
*/
public String getFileName() {
return name().toLowerCase().replace('_', '-') + "-" + this.version;
}
public List<URL> getUrls() {
return this.urls;
}
public String getVersion() {
return this.version;
String getMavenRepoPath() {
return this.mavenRepoPath;
}
public byte[] getChecksum() {
return this.checksum;
}
public boolean checksumMatches(byte[] hash) {
return Arrays.equals(this.checksum, hash);
}
public List<Relocation> getRelocations() {
return this.relocations;
}
/**
* Creates a {@link MessageDigest} suitable for computing the checksums
* of dependencies.
*
* @return the digest
*/
public static MessageDigest createDigest() {
try {
return MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
/*
public static void main(String[] args) {
Dependency[] dependencies = values();
DependencyRepository[] repos = DependencyRepository.values();
java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newCachedThreadPool();
for (Dependency dependency : dependencies) {
for (DependencyRepository repo : repos) {
pool.submit(() -> {
try {
byte[] hash = createDigest().digest(repo.downloadRaw(dependency));
if (!dependency.checksumMatches(hash)) {
System.out.println("NO MATCH - " + repo.name() + " - " + dependency.name() + ": " + Base64.getEncoder().encodeToString(hash));
} else {
System.out.println("OK - " + repo.name() + " - " + dependency.name());
}
} catch (Exception e) {
e.printStackTrace();
}
});
}
}
pool.shutdown();
try {
pool.awaitTermination(1, java.util.concurrent.TimeUnit.HOURS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
*/
}

View File

@ -0,0 +1,48 @@
/*
* This file is part of LuckPerms, licensed under the MIT License.
*
* Copyright (c) lucko (Luck) <luck@lucko.me>
* Copyright (c) contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package me.lucko.luckperms.common.dependencies;
/**
* Exception thrown if a dependency cannot be downloaded.
*/
public class DependencyDownloadException extends Exception {
public DependencyDownloadException() {
}
public DependencyDownloadException(String message) {
super(message);
}
public DependencyDownloadException(String message, Throwable cause) {
super(message, cause);
}
public DependencyDownloadException(Throwable cause) {
super(cause);
}
}

View File

@ -26,7 +26,6 @@
package me.lucko.luckperms.common.dependencies;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
import me.lucko.luckperms.common.dependencies.classloader.IsolatedClassLoader;
import me.lucko.luckperms.common.dependencies.relocation.Relocation;
@ -35,62 +34,45 @@ import me.lucko.luckperms.common.plugin.LuckPermsPlugin;
import me.lucko.luckperms.common.storage.StorageType;
import me.lucko.luckperms.common.util.MoreFiles;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.CountDownLatch;
/**
* Responsible for loading runtime dependencies.
* Loads and manages runtime dependencies for the plugin.
*/
public class DependencyManager {
private final LuckPermsPlugin plugin;
private final MessageDigest digest;
private final DependencyRegistry registry;
private final Path libsDirectory;
/** The plugin instance */
private final LuckPermsPlugin plugin;
/** A registry containing plugin specific behaviour for dependencies. */
private final DependencyRegistry registry;
/** The path where library jars are cached. */
private final Path cacheDirectory;
/** A map of dependencies which have already been loaded. */
private final EnumMap<Dependency, Path> loaded = new EnumMap<>(Dependency.class);
/** A map of isolated classloaders which have been created. */
private final Map<ImmutableSet<Dependency>, IsolatedClassLoader> loaders = new HashMap<>();
private RelocationHandler relocationHandler = null;
/** Cached relocation handler instance. */
private @MonotonicNonNull RelocationHandler relocationHandler = null;
public DependencyManager(LuckPermsPlugin plugin) {
this.plugin = plugin;
try {
this.digest = MessageDigest.getInstance("SHA-256");
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
this.registry = new DependencyRegistry(plugin);
this.libsDirectory = this.plugin.getBootstrap().getDataDirectory().resolve("libs");
try {
MoreFiles.createDirectoriesIfNotExists(this.libsDirectory);
} catch (IOException e) {
throw new RuntimeException("Unable to create libs directory", e);
}
Path oldLibsDirectory = this.plugin.getBootstrap().getDataDirectory().resolve("lib");
if (Files.exists(oldLibsDirectory)) {
try {
MoreFiles.deleteDirectory(oldLibsDirectory);
} catch (IOException e) {
e.printStackTrace();
}
}
this.cacheDirectory = setupCacheDirectory(plugin);
}
private synchronized RelocationHandler getRelocationHandler() {
@ -137,157 +119,102 @@ public class DependencyManager {
}
public void loadDependencies(Set<Dependency> dependencies) {
// create a list of file sources
List<Source> sources = new ArrayList<>();
CountDownLatch latch = new CountDownLatch(dependencies.size());
// obtain a file for each of the dependencies
for (Dependency dependency : dependencies) {
if (this.loaded.containsKey(dependency)) {
continue;
}
try {
Path file = downloadDependency(dependency);
sources.add(new Source(dependency, file));
} catch (Throwable e) {
this.plugin.getLogger().severe("Exception whilst downloading dependency " + dependency.name());
e.printStackTrace();
}
this.plugin.getBootstrap().getScheduler().async().execute(() -> {
try {
loadDependency(dependency);
} catch (Throwable e) {
this.plugin.getLogger().severe("Unable to load dependency " + dependency.name() + ".");
e.printStackTrace();
} finally {
latch.countDown();
}
});
}
// apply any remapping rules to the files
List<Source> remappedJars = new ArrayList<>(sources.size());
for (Source source : sources) {
try {
// apply remap rules
List<Relocation> relocations = new ArrayList<>(source.dependency.getRelocations());
this.registry.applyRelocationSettings(source.dependency, relocations);
if (relocations.isEmpty()) {
remappedJars.add(source);
continue;
}
Path input = source.file;
Path output = this.libsDirectory.resolve(source.dependency.getFileName() + "-remapped.jar");
// if the remapped file exists already, just use that.
if (Files.exists(output)) {
remappedJars.add(new Source(source.dependency, output));
continue;
}
// init the relocation handler
RelocationHandler relocationHandler = getRelocationHandler();
// attempt to remap the jar.
relocationHandler.remap(input, output, relocations);
remappedJars.add(new Source(source.dependency, output));
} catch (Throwable e) {
this.plugin.getLogger().severe("Unable to remap the source file '" + source.dependency.name() + "'.");
e.printStackTrace();
}
}
// load each of the jars
for (Source source : remappedJars) {
if (!DependencyRegistry.shouldAutoLoad(source.dependency)) {
this.loaded.put(source.dependency, source.file);
continue;
}
try {
this.plugin.getBootstrap().getPluginClassLoader().addJarToClasspath(source.file);
this.loaded.put(source.dependency, source.file);
} catch (Throwable e) {
this.plugin.getLogger().severe("Failed to load dependency jar '" + source.file.getFileName().toString() + "'.");
e.printStackTrace();
}
try {
latch.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
private Path downloadDependency(Dependency dependency) throws Exception {
Path file = this.libsDirectory.resolve(dependency.getFileName() + ".jar");
private void loadDependency(Dependency dependency) throws Exception {
if (this.loaded.containsKey(dependency)) {
return;
}
Path file = remapDependency(dependency, downloadDependency(dependency));
this.loaded.put(dependency, file);
if (this.registry.shouldAutoLoad(dependency)) {
this.plugin.getBootstrap().getPluginClassLoader().addJarToClasspath(file);
}
}
private Path downloadDependency(Dependency dependency) throws DependencyDownloadException {
Path file = this.cacheDirectory.resolve(dependency.getFileName() + ".jar");
// if the file already exists, don't attempt to re-download it.
if (Files.exists(file)) {
return file;
}
boolean success = false;
Exception lastError = null;
// getUrls returns two possible sources of the dependency.
// [0] is a mirror of Maven Central, used to reduce load on central. apparently they don't like being used as a CDN
// [1] is Maven Central itself
// side note: the relative "security" of the mirror is less than central, but it actually doesn't matter.
// we compare the downloaded file against a checksum here, so even if the mirror became compromised, RCE wouldn't be possible.
// if the mirror download doesn't match the checksum, we just try maven central instead.
List<URL> urls = dependency.getUrls();
for (int i = 0; i < urls.size() && !success; i++) {
URL url = urls.get(i);
DependencyDownloadException lastError = null;
// attempt to download the dependency from each repo in order.
for (DependencyRepository repo : DependencyRepository.values()) {
try {
URLConnection connection = url.openConnection();
// i == 0 when we're trying to use the mirror repo.
// set some timeout properties so when/if this repository goes offline, we quickly fallback to central.
if (i == 0) {
connection.setRequestProperty("User-Agent", "luckperms");
connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(5));
connection.setReadTimeout((int) TimeUnit.SECONDS.toMillis(10));
}
try (InputStream in = connection.getInputStream()) {
// download the jar content
byte[] bytes = ByteStreams.toByteArray(in);
if (bytes.length == 0) {
throw new RuntimeException("Empty stream");
}
// compute a hash for the downloaded file
byte[] hash = this.digest.digest(bytes);
// ensure the hash matches the expected checksum
if (!Arrays.equals(hash, dependency.getChecksum())) {
throw new RuntimeException("Downloaded file had an invalid hash. " +
"Expected: " + Base64.getEncoder().encodeToString(dependency.getChecksum()) + " " +
"Actual: " + Base64.getEncoder().encodeToString(hash));
}
// if the checksum matches, save the content to disk
Files.write(file, bytes);
success = true;
}
} catch (Exception e) {
repo.download(dependency, file);
return file;
} catch (DependencyDownloadException e) {
lastError = e;
}
}
if (!success) {
throw new RuntimeException("Unable to download", lastError);
}
// ensure the file saved correctly
if (!Files.exists(file)) {
throw new IllegalStateException("File not present: " + file.toString());
} else {
return file;
}
throw Objects.requireNonNull(lastError);
}
private static final class Source {
private final Dependency dependency;
private final Path file;
private Path remapDependency(Dependency dependency, Path normalFile) throws Exception {
List<Relocation> rules = new ArrayList<>(dependency.getRelocations());
this.registry.applyRelocationSettings(dependency, rules);
private Source(Dependency dependency, Path file) {
this.dependency = dependency;
this.file = file;
if (rules.isEmpty()) {
return normalFile;
}
Path remappedFile = this.cacheDirectory.resolve(dependency.getFileName() + "-remapped.jar");
// if the remapped source exists already, just use that.
if (Files.exists(remappedFile)) {
return remappedFile;
}
getRelocationHandler().remap(normalFile, remappedFile, rules);
return remappedFile;
}
private static Path setupCacheDirectory(LuckPermsPlugin plugin) {
Path cacheDirectory = plugin.getBootstrap().getDataDirectory().resolve("libs");
try {
MoreFiles.createDirectoriesIfNotExists(cacheDirectory);
} catch (IOException e) {
throw new RuntimeException("Unable to create libs directory", e);
}
Path oldCacheDirectory = plugin.getBootstrap().getDataDirectory().resolve("lib");
if (Files.exists(oldCacheDirectory)) {
try {
MoreFiles.deleteDirectory(oldCacheDirectory);
} catch (IOException e) {
e.printStackTrace();
}
}
return cacheDirectory;
}
}

View File

@ -41,6 +41,9 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
/**
* Applies LuckPerms specific behaviour for {@link Dependency}s.
*/
public class DependencyRegistry {
private static final ListMultimap<StorageType, Dependency> STORAGE_DEPENDENCIES = ImmutableListMultimap.<StorageType, Dependency>builder()
@ -103,20 +106,7 @@ public class DependencyRegistry {
}
}
private static boolean classExists(String className) {
try {
Class.forName(className);
return true;
} catch (ClassNotFoundException e) {
return false;
}
}
private static boolean slf4jPresent() {
return classExists("org.slf4j.Logger") && classExists("org.slf4j.LoggerFactory");
}
public static boolean shouldAutoLoad(Dependency dependency) {
public boolean shouldAutoLoad(Dependency dependency) {
switch (dependency) {
// all used within 'isolated' classloaders, and are therefore not
// relocated.
@ -131,4 +121,17 @@ public class DependencyRegistry {
}
}
private static boolean classExists(String className) {
try {
Class.forName(className);
return true;
} catch (ClassNotFoundException e) {
return false;
}
}
private static boolean slf4jPresent() {
return classExists("org.slf4j.Logger") && classExists("org.slf4j.LoggerFactory");
}
}

View File

@ -0,0 +1,153 @@
/*
* This file is part of LuckPerms, licensed under the MIT License.
*
* Copyright (c) lucko (Luck) <luck@lucko.me>
* Copyright (c) contributors
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package me.lucko.luckperms.common.dependencies;
import com.google.common.io.ByteStreams;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Base64;
import java.util.concurrent.TimeUnit;
/**
* Represents a repository which contains {@link Dependency}s.
*/
public enum DependencyRepository {
/**
* Maven Central mirror repository.
*
* <p>This is used to reduce the load on repo.maven.org - I'm told they
* don't like being used as a CDN.</p>
*/
// Please ask me (@lucko) before using this mirror in your own project.
LUCK_MIRROR("https://nexus.lucko.me/repository/maven-central/") {
@Override
protected URLConnection openConnection(Dependency dependency) throws IOException {
URLConnection connection = super.openConnection(dependency);
// Tell nexus who we are
connection.setRequestProperty("User-Agent", "luckperms");
// Set a connect/read timeout, so if the mirror goes offline we can fallback
// to Maven Central within a reasonable time.
connection.setConnectTimeout((int) TimeUnit.SECONDS.toMillis(5));
connection.setReadTimeout((int) TimeUnit.SECONDS.toMillis(10));
return connection;
}
},
/**
* Maven Central.
*/
MAVEN_CENTRAL("https://repo1.maven.org/maven2/");
private final String url;
DependencyRepository(String url) {
this.url = url;
}
/**
* Opens a connection to the given {@code dependency}.
*
* @param dependency the dependency to download
* @return the connection
* @throws IOException if unable to open a connection
*/
protected URLConnection openConnection(Dependency dependency) throws IOException {
URL dependencyUrl = new URL(this.url + dependency.getMavenRepoPath());
return dependencyUrl.openConnection();
}
/**
* Downloads the raw bytes of the {@code dependency}.
*
* @param dependency the dependency to download
* @return the downloaded bytes
* @throws DependencyDownloadException if unable to download
*/
public byte[] downloadRaw(Dependency dependency) throws DependencyDownloadException {
try {
URLConnection connection = openConnection(dependency);
try (InputStream in = connection.getInputStream()) {
byte[] bytes = ByteStreams.toByteArray(in);
if (bytes.length == 0) {
throw new DependencyDownloadException("Empty stream");
}
return bytes;
}
} catch (Exception e) {
throw new DependencyDownloadException(e);
}
}
/**
* Downloads the raw bytes of the {@code dependency}, and ensures the downloaded
* bytes match the checksum.
*
* @param dependency the dependency to download
* @return the downloaded bytes
* @throws DependencyDownloadException if unable to download
*/
public byte[] download(Dependency dependency) throws DependencyDownloadException {
byte[] bytes = downloadRaw(dependency);
// compute a hash for the downloaded file
byte[] hash = Dependency.createDigest().digest(bytes);
// ensure the hash matches the expected checksum
if (!dependency.checksumMatches(hash)) {
throw new DependencyDownloadException("Downloaded file had an invalid hash. " +
"Expected: " + Base64.getEncoder().encodeToString(dependency.getChecksum()) + " " +
"Actual: " + Base64.getEncoder().encodeToString(hash));
}
return bytes;
}
/**
* Downloads the the {@code dependency} to the {@code file}, ensuring the
* downloaded bytes match the checksum.
*
* @param dependency the dependency to download
* @param file the file to write to
* @throws DependencyDownloadException if unable to download
*/
public void download(Dependency dependency, Path file) throws DependencyDownloadException {
try {
Files.write(file, download(dependency));
} catch (IOException e) {
throw new DependencyDownloadException(e);
}
}
}