diff --git a/.github/workflows/bidi-checker.yml b/.github/workflows/bidi-checker.yml index 622549dbf9..a059875b68 100644 --- a/.github/workflows/bidi-checker.yml +++ b/.github/workflows/bidi-checker.yml @@ -25,6 +25,6 @@ jobs: id: bidi_check uses: HL7/bidi-checker-action@v1.9 env: - IGNORE: i18n-coverage-table\.png$|dummy-package.tgz$ + IGNORE: i18n-coverage-table\.png$|dummy-package\.tgz$|dummy-package-no-index\.tgz$ - name: Get the output time run: echo "The time was ${{ steps.bidi_check.outputs.time }}" diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java index 44bb7729c3..d700e6e08b 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/Utilities.java @@ -69,7 +69,6 @@ WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWIS import org.apache.commons.io.FileUtils; import org.hl7.fhir.exceptions.FHIRException; import org.hl7.fhir.utilities.FileNotifier.FileNotifier2; -import org.hl7.fhir.utilities.Utilities.CaseInsensitiveSorter; import org.hl7.fhir.utilities.filesystem.CSFile; import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; import org.hl7.fhir.utilities.settings.FhirSettings; @@ -451,6 +450,28 @@ public static String asHtmlBr(String prefix, List strings) { return s.toString(); } + /** + * Delete a directory atomically by first renaming it to a temp directory in + * its parent, and then deleting its contents. + * + * @param path The directory to delete. + */ + public static void atomicDeleteDirectory(String path) throws IOException { + + File directory = ManagedFileAccess.file(path); + + String tempDirectoryPath = generateUniqueRandomUUIDPath(directory.getParent()); + File tempDirectory = ManagedFileAccess.file(tempDirectoryPath); + if (!directory.renameTo(tempDirectory)) { + throw new IOException("Unable to rename directory " + path + " to " + tempDirectory +" for atomic delete"); + } + clearDirectory(tempDirectory.getAbsolutePath()); + if (!tempDirectory.delete()) { + throw new IOException("Unable to delete temp directory " + tempDirectory + " when atomically deleting " + path); + } + } + + public static void clearDirectory(String folder, String... exemptions) throws IOException { File dir = ManagedFileAccess.file(folder); if (dir.exists()) { @@ -472,6 +493,21 @@ public static void clearDirectory(String folder, String... exemptions) throws IO } } + public static String generateUniqueRandomUUIDPath(String path) throws IOException { + String randomUUIDPath = null; + + while (randomUUIDPath == null) { + final String uuid = UUID.randomUUID().toString().toLowerCase(); + final String pathCandidate = Utilities.path(path, uuid); + + if (!ManagedFileAccess.file(pathCandidate).exists()) { + randomUUIDPath = pathCandidate; + } + } + + return randomUUIDPath; + } + public static File createDirectory(String path) throws IOException { ManagedFileAccess.csfile(path).mkdirs(); return ManagedFileAccess.file(path); diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java index 13a387eee8..81eb9cca5f 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/BasePackageCacheManager.java @@ -186,6 +186,8 @@ public InputStreamWithSrc(InputStream stream, String url, String version) { } } + + @Override public NpmPackage loadPackage(String idAndVer) throws FHIRException, IOException { return loadPackage(idAndVer, null); } diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/CIBuildClient.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/CIBuildClient.java new file mode 100644 index 0000000000..f86a5093b3 --- /dev/null +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/CIBuildClient.java @@ -0,0 +1,232 @@ +package org.hl7.fhir.utilities.npm; + +import lombok.Getter; +import org.hl7.fhir.exceptions.FHIRException; +import org.hl7.fhir.utilities.TextFile; +import org.hl7.fhir.utilities.Utilities; +import org.hl7.fhir.utilities.http.HTTPResult; +import org.hl7.fhir.utilities.http.ManagedWebAccess; +import org.hl7.fhir.utilities.json.model.JsonArray; +import org.hl7.fhir.utilities.json.model.JsonElement; +import org.hl7.fhir.utilities.json.model.JsonObject; +import org.hl7.fhir.utilities.json.parser.JsonParser; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; + +public class CIBuildClient { + + private static final String DEFAULT_ROOT_URL = "https://build.fhir.org"; + private static final long DEFAULT_CI_QUERY_INTERVAL = 1000 * 60 * 60; + private final long ciQueryInterval; + + @Getter + private long ciLastQueriedTimeStamp = 0; + + @Getter + private JsonArray ciBuildInfo; + + @Getter + private final String rootUrl; + + /** + * key = packageId + * value = url of built package on https://build.fhir.org/ig/ + **/ + private final Map ciPackageUrls = new HashMap<>(); + + private final boolean silent; + + public CIBuildClient() { + this(DEFAULT_ROOT_URL, DEFAULT_CI_QUERY_INTERVAL, false); + } + + public CIBuildClient(String rootUrl, long ciQueryInterval, boolean silent) { + this.rootUrl = rootUrl; + this.ciQueryInterval = ciQueryInterval; + this.silent = silent; + } + + String getPackageId(String canonical) { + if (canonical == null) { + return null; + } + checkCIServerQueried(); + if (ciBuildInfo != null) { + for (JsonElement n : ciBuildInfo) { + JsonObject o = (JsonObject) n; + if (canonical.equals(o.asString("url"))) { + return o.asString("package-id"); + } + } + for (JsonElement n : ciBuildInfo) { + JsonObject o = (JsonObject) n; + if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) { + return o.asString("package-id"); + } + } + } + return null; + } + + String getPackageUrl(String packageId) { + checkCIServerQueried(); + for (JsonObject o : ciBuildInfo.asJsonObjects()) { + if (packageId.equals(o.asString("package-id"))) { + return o.asString("url"); + } + } + return null; + } + + public boolean isCurrent(String id, NpmPackage npmPackage) throws IOException { + checkCIServerQueried(); + String packageManifestUrl = ciPackageUrls.get(id); + JsonObject packageManifestJson = JsonParser.parseObjectFromUrl(Utilities.pathURL(packageManifestUrl, "package.manifest.json")); + String currentDate = packageManifestJson.asString("date"); + String packageDate = npmPackage.date(); + return currentDate.equals(packageDate); // nup, we need a new copy + } + + BasePackageCacheManager.InputStreamWithSrc loadFromCIBuild(String id, String branch) { + checkCIServerQueried(); + + if (ciPackageUrls.containsKey(id)) { + String packageBaseUrl = ciPackageUrls.get(id); + if (branch == null) { + InputStream stream; + try { + stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "package.tgz")); + } catch (Exception e) { + stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "branches", "main", "package.tgz")); + } + return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(packageBaseUrl, "package.tgz"), "current"); + } else { + InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(packageBaseUrl, "branches", branch, "package.tgz")); + return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(packageBaseUrl, "branches", branch, "package.tgz"), "current$" + branch); + } + } else if (id.startsWith("hl7.fhir.r6")) { + InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(rootUrl, id + ".tgz")); + return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(rootUrl, id + ".tgz"), "current"); + } else if (id.startsWith("hl7.fhir.uv.extensions.")) { + InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(rootUrl + "/ig/HL7/fhir-extensions/", id + ".tgz")); + return new BasePackageCacheManager.InputStreamWithSrc(stream, Utilities.pathURL(rootUrl + "/ig/HL7/fhir-extensions/", id + ".tgz"), "current"); + } else { + throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciPackageUrls + ")"); + } + } + + private InputStream fetchFromUrlSpecific(String source) throws FHIRException { + try { + HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), source); + res.checkThrowException(); + return new ByteArrayInputStream(res.getContent()); + } catch (Exception e) { + throw new FHIRException("Unable to fetch: " + e.getMessage(), e); + } + } + + private void checkCIServerQueried() { + if (System.currentTimeMillis() - ciLastQueriedTimeStamp > ciQueryInterval) { + try { + updateFromCIServer(); + } catch (Exception e) { + try { + // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server + Thread.sleep(1000); + updateFromCIServer(); + } catch (Exception e2) { + if (!silent) { + System.out.println("Error connecting to build server - running without build (" + e2.getMessage() + ")"); + } + } + } + } + } + + private void updateFromCIServer() throws IOException { + HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), rootUrl + "/ig/qas.json?nocache=" + System.currentTimeMillis()); + res.checkThrowException(); + + ciBuildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent())); + + List builds = new ArrayList<>(); + + for (JsonElement n : ciBuildInfo) { + JsonObject j = (JsonObject) n; + if (j.has("url") && j.has("package-id") && j.asString("package-id").contains(".")) { + String packageUrl = j.asString("url"); + if (packageUrl.contains("/ImplementationGuide/")) + packageUrl = packageUrl.substring(0, packageUrl.indexOf("/ImplementationGuide/")); + builds.add(new BuildRecord(packageUrl, j.asString("package-id"), getRepo(j.asString("repo")), readDate(j.asString("date")))); + } + } + Collections.sort(builds, new BuildRecordSorter()); + for (BuildRecord build : builds) { + if (!ciPackageUrls.containsKey(build.getPackageId())) { + ciPackageUrls.put(build.getPackageId(), rootUrl + "/ig/" + build.getRepo()); + } + } + ciLastQueriedTimeStamp = System.currentTimeMillis(); + } + + private String getRepo(String path) { + String[] p = path.split("/"); + return p[0] + "/" + p[1]; + } + + private Date readDate(String s) { + SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US")); + try { + return sdf.parse(s); + } catch (ParseException e) { + e.printStackTrace(); + return new Date(); + } + } + + public static class BuildRecord { + + private final String url; + private final String packageId; + private final String repo; + private final Date date; + + public BuildRecord(String url, String packageId, String repo, Date date) { + super(); + this.url = url; + this.packageId = packageId; + this.repo = repo; + this.date = date; + } + + public String getUrl() { + return url; + } + + public String getPackageId() { + return packageId; + } + + public String getRepo() { + return repo; + } + + public Date getDate() { + return date; + } + + } + + public static class BuildRecordSorter implements Comparator { + + @Override + public int compare(BuildRecord arg0, BuildRecord arg1) { + return arg1.date.compareTo(arg0.date); + } + } +} diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java index 58c41ad510..9681a31e84 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/FilesystemPackageCacheManager.java @@ -2,9 +2,6 @@ import java.io.*; -import java.text.ParseException; -import java.text.SimpleDateFormat; - import java.util.*; import javax.annotation.Nonnull; @@ -22,8 +19,6 @@ import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; import org.hl7.fhir.utilities.http.HTTPResult; import org.hl7.fhir.utilities.http.ManagedWebAccess; -import org.hl7.fhir.utilities.json.model.JsonArray; -import org.hl7.fhir.utilities.json.model.JsonElement; import org.hl7.fhir.utilities.json.model.JsonObject; import org.hl7.fhir.utilities.json.parser.JsonParser; import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry; @@ -95,13 +90,14 @@ public interface IPackageProvider { private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); private static final String CACHE_VERSION = "3"; // second version - see wiki page + @Getter + private final CIBuildClient ciBuildClient; + @Nonnull private final File cacheFolder; private final List temporaryPackages = new ArrayList<>(); - private boolean buildLoaded = false; - private final Map ciList = new HashMap<>(); - private JsonArray buildInfo; + private boolean suppressErrors; @Setter @@ -117,6 +113,10 @@ public static class Builder { @Getter private final List packageServers; + @With + @Getter + private final CIBuildClient ciBuildClient; + @With @Getter private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; @@ -125,11 +125,13 @@ public Builder() throws IOException { this.cacheFolder = getUserCacheFolder(); this.packageServers = getPackageServersFromFHIRSettings(); this.lockParameters = null; + this.ciBuildClient = new CIBuildClient(); } - private Builder(File cacheFolder, List packageServers, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { + private Builder(File cacheFolder, List packageServers, CIBuildClient ciBuildClient, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { this.cacheFolder = cacheFolder; this.packageServers = packageServers; + this.ciBuildClient = ciBuildClient; this.lockParameters = lockParameters; } @@ -163,7 +165,7 @@ public Builder withCacheFolder(String cacheFolderPath) throws IOException { if (!cacheFolder.exists()) { throw new FHIRException("The folder '" + cacheFolder + "' could not be found"); } - return new Builder(cacheFolder, this.packageServers, this.lockParameters); + return new Builder(cacheFolder, this.packageServers, this.ciBuildClient, this.lockParameters); } public Builder withSystemCacheFolder() throws IOException { @@ -173,11 +175,11 @@ public Builder withSystemCacheFolder() throws IOException { } else { systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages")); } - return new Builder(systemCacheFolder, this.packageServers, this.lockParameters); + return new Builder(systemCacheFolder, this.packageServers, this.ciBuildClient, this.lockParameters); } public Builder withTestingCacheFolder() throws IOException { - return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.lockParameters); + return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.ciBuildClient, this.lockParameters); } public FilesystemPackageCacheManager build() throws IOException { @@ -191,13 +193,14 @@ public FilesystemPackageCacheManager build() throws IOException { throw e; } } - return new FilesystemPackageCacheManager(cacheFolder, packageServers, locks, lockParameters); + return new FilesystemPackageCacheManager(cacheFolder, packageServers, ciBuildClient, locks, lockParameters); } } - private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List packageServers, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { + private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List packageServers, CIBuildClient ciBuildClient, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { super(packageServers); this.cacheFolder = cacheFolder; + this.ciBuildClient = ciBuildClient; this.locks = locks; this.lockParameters = lockParameters; prepareCacheFolder(); @@ -280,27 +283,6 @@ private void deleteOldTempDirectories() throws IOException { } } - - private void initPackageServers() { - myPackageServers.addAll(getConfiguredServers()); - if (!isIgnoreDefaultPackageServers()) { - myPackageServers.addAll(getDefaultServers()); - } - } - - protected boolean isIgnoreDefaultPackageServers() { - return FhirSettings.isIgnoreDefaultPackageServers(); - } - - @Nonnull - protected List getDefaultServers() { - return PackageServer.defaultServers(); - } - - protected List getConfiguredServers() { - return PackageServer.getConfiguredServers(); - } - public String getFolder() { return cacheFolder.getAbsolutePath(); } @@ -312,16 +294,7 @@ private NpmPackage loadPackageInfo(String path) throws IOException { private void clearCache() throws IOException { for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { if (f.isDirectory()) { - Utilities.clearDirectory(f.getAbsolutePath()); - try { - FileUtils.deleteDirectory(f); - } catch (Exception e1) { - try { - FileUtils.deleteDirectory(f); - } catch (Exception e2) { - // just give up - } - } + Utilities.atomicDeleteDirectory(f.getAbsolutePath()); } else if (!f.getName().equals("packages.ini")) { FileUtils.forceDelete(f); @@ -449,12 +422,10 @@ public void removePackage(String id, String version) throws IOException { locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { String f = Utilities.path(cacheFolder, id + "#" + version); - File ff = ManagedFileAccess.file(f); - if (ff.exists()) { - Utilities.clearDirectory(f); - ff.delete(); - } - + File ff = ManagedFileAccess.file(f); + if (ff.exists()) { + Utilities.atomicDeleteDirectory(f); + } return null; }, lockParameters); } @@ -561,25 +532,42 @@ private String findPackageFolder(String id, String version) throws IOException { public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException { checkValidVersionString(version, id); return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { - String uuid = UUID.randomUUID().toString().toLowerCase(); - String tempDir = Utilities.path(cacheFolder, uuid); - NpmPackage npm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); + String tempDir = Utilities.generateUniqueRandomUUIDPath(cacheFolder.getAbsolutePath()); + + NpmPackage extractedNpm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); log(""); log("Installing " + id + "#" + version); - if ((npm.name() != null && id != null && !id.equalsIgnoreCase(npm.name()))) { + if ((extractedNpm.name() != null && id != null && !id.equalsIgnoreCase(extractedNpm.name()))) { if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around - throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name()); + throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + extractedNpm.name()); } } - - NpmPackage npmPackage = null; + NpmPackage npmPackage; String packageRoot = Utilities.path(cacheFolder, id + "#" + version); try { - // ok, now we have a lock on it... check if something created it while we were waiting + if (!id.equals(extractedNpm.getNpm().asString("name")) || !version.equals(extractedNpm.getNpm().asString("version"))) { + if (!id.equals(extractedNpm.getNpm().asString("name"))) { + extractedNpm.getNpm().add("original-name", extractedNpm.getNpm().asString("name")); + extractedNpm.getNpm().remove("name"); + extractedNpm.getNpm().add("name", id); + } + if (!version.equals(extractedNpm.getNpm().asString("version"))) { + extractedNpm.getNpm().add("original-version", extractedNpm.getNpm().asString("version")); + extractedNpm.getNpm().remove("version"); + extractedNpm.getNpm().add("version", version); + } + TextFile.stringToFile(JsonParser.compose(extractedNpm.getNpm(), true), Utilities.path(tempDir, "package", "package.json")); + } + + final NpmPackage tempPackage = loadPackageInfo(tempDir); + if (tempPackage != null && !tempPackage.isIndexed()) { + tempPackage.checkIndexed(packageRoot); + } + if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) { Utilities.createDirectory(packageRoot); try { @@ -587,30 +575,18 @@ public NpmPackage addPackageToCache(final String id, final String version, final } catch (Throwable t) { log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later"); } + Utilities.renameDirectory(tempDir, packageRoot); + npmPackage = loadPackageInfo(packageRoot); + log(" done."); } else { Utilities.clearDirectory(tempDir); ManagedFileAccess.file(tempDir).delete(); } - if (!id.equals(npm.getNpm().asString("name")) || !version.equals(npm.getNpm().asString("version"))) { - if (!id.equals(npm.getNpm().asString("name"))) { - npm.getNpm().add("original-name", npm.getNpm().asString("name")); - npm.getNpm().remove("name"); - npm.getNpm().add("name", id); - } - if (!version.equals(npm.getNpm().asString("version"))) { - npm.getNpm().add("original-version", npm.getNpm().asString("version")); - npm.getNpm().remove("version"); - npm.getNpm().add("version", version); - } - TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json")); - } npmPackage = loadPackageInfo(packageRoot); - if (npmPackage != null && !npmPackage.isIndexed()) { - npmPackage.checkIndexed(packageRoot); - } + } catch (Exception e) { try { // don't leave a half extracted package behind @@ -637,9 +613,8 @@ private void log(String s) { public String getPackageUrl(String packageId) throws IOException { String result = super.getPackageUrl(packageId); if (result == null) { - result = getPackageUrlFromBuildList(packageId); + result = ciBuildClient.getPackageUrl(packageId); } - return result; } @@ -688,7 +663,7 @@ public NpmPackage loadPackage(String id, String version) throws FHIRException, I NpmPackage p = loadPackageFromCacheOnly(id, version); if (p != null) { if ("current".equals(version)) { - p = checkCurrency(id, p); + p = nullIfNotCurrentPackage(id, p); } if (p != null) return p; @@ -696,7 +671,7 @@ public NpmPackage loadPackage(String id, String version) throws FHIRException, I if ("dev".equals(version)) { p = loadPackageFromCacheOnly(id, "current"); - p = checkCurrency(id, p); + p = nullIfNotCurrentPackage(id, p); if (p != null) return p; version = "current"; @@ -713,7 +688,7 @@ public NpmPackage loadPackage(String id, String version) throws FHIRException, I source = fetchSourceFromUrlSpecific(version); } else if ("current".equals(version) || (version != null && version.startsWith("current$"))) { // special case - fetch from ci-build server - source = loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null); + source = ciBuildClient.loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null); } else { source = loadFromPackageServer(id, version); } @@ -740,41 +715,7 @@ private InputStream fetchFromUrlSpecific(String source, boolean optional) throws } } - private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException { - checkBuildLoaded(); - if (ciList.containsKey(id)) { - if (branch == null) { - InputStream stream; - try { - stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false); - } catch (Exception e) { - stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false); - } - return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current"); - } else { - InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false); - return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$" + branch); - } - } else if (id.startsWith("hl7.fhir.r6")) { - InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false); - return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current"); - } else if (id.startsWith("hl7.fhir.uv.extensions.")) { - InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org/ig/HL7/fhir-extensions/", id + ".tgz"), false); - return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org/ig/HL7/fhir-extensions/", id + ".tgz"), "current"); - } else { - throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciList + ")"); - } - } - private String getPackageUrlFromBuildList(String packageId) throws IOException { - checkBuildLoaded(); - for (JsonObject o : buildInfo.asJsonObjects()) { - if (packageId.equals(o.asString("package-id"))) { - return o.asString("url"); - } - } - return null; - } @Override public String getPackageId(String canonicalUrl) throws IOException { @@ -785,7 +726,7 @@ public String getPackageId(String canonicalUrl) throws IOException { } if (retVal == null) { - retVal = getPackageIdFromBuildList(canonicalUrl); + retVal = ciBuildClient.getPackageId(canonicalUrl); } return retVal; @@ -809,100 +750,21 @@ public String findCanonicalInLocalCache(String canonicalUrl) { // ========================= Package Mgmt API ======================================================================= - private String getPackageIdFromBuildList(String canonical) { - if (canonical == null) { - return null; - } - checkBuildLoaded(); - if (buildInfo != null) { - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (canonical.equals(o.asString("url"))) { - return o.asString("package-id"); - } - } - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) { - return o.asString("package-id"); - } - } - } - return null; - } - - private NpmPackage checkCurrency(String id, NpmPackage p) { - checkBuildLoaded(); + /** + * Checks https:// + * + * @param id + * @param npmPackage + * @return + */ + private NpmPackage nullIfNotCurrentPackage(String id, NpmPackage npmPackage) { // special case: current versions roll over, and we have to check their currency try { - String url = ciList.get(id); - JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json")); - String currDate = json.asString("date"); - String packDate = p.date(); - if (!currDate.equals(packDate)) { - return null; // nup, we need a new copy - } + return ciBuildClient.isCurrent(id, npmPackage) ? npmPackage : null; } catch (Exception e) { log("Unable to check package currency: " + id + ": " + id); } - return p; - } - - private void checkBuildLoaded() { - if (!buildLoaded) { - try { - loadFromBuildServer(); - } catch (Exception e) { - try { - // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server - Thread.sleep(1000); - loadFromBuildServer(); - } catch (Exception e2) { - log("Error connecting to build server - running without build (" + e2.getMessage() + ")"); - } - } - } - } - - private void loadFromBuildServer() throws IOException { - HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), "https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis()); - res.checkThrowException(); - - buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent())); - - List builds = new ArrayList<>(); - - for (JsonElement n : buildInfo) { - JsonObject o = (JsonObject) n; - if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) { - String u = o.asString("url"); - if (u.contains("/ImplementationGuide/")) - u = u.substring(0, u.indexOf("/ImplementationGuide/")); - builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date")))); - } - } - Collections.sort(builds, new BuildRecordSorter()); - for (BuildRecord bld : builds) { - if (!ciList.containsKey(bld.getPackageId())) { - ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo()); - } - } - buildLoaded = true; - } - - private String getRepo(String path) { - String[] p = path.split("\\/"); - return p[0] + "/" + p[1]; - } - - private Date readDate(String s) { - SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US")); - try { - return sdf.parse(s); - } catch (ParseException e) { - e.printStackTrace(); - return new Date(); - } + return npmPackage; } // ----- the old way, from before package server, while everything gets onto the package server @@ -910,7 +772,7 @@ private InputStreamWithSrc fetchTheOldWay(String id, String v) { String url = getUrlForPackage(id); if (url == null) { try { - url = getPackageUrlFromBuildList(id); + url = ciBuildClient.getPackageUrl(id); } catch (Exception ignored) { } @@ -954,7 +816,7 @@ private String fetchVersionTheOldWay(String id) throws IOException { String url = getUrlForPackage(id); if (url == null) { try { - url = getPackageUrlFromBuildList(id); + url = ciBuildClient.getPackageUrl(id); } catch (Exception e) { url = null; } @@ -995,46 +857,9 @@ public interface CacheLockFunction { T get() throws IOException; } - public class BuildRecordSorter implements Comparator { - - @Override - public int compare(BuildRecord arg0, BuildRecord arg1) { - return arg1.date.compareTo(arg0.date); - } - } - - public class BuildRecord { - private final String url; - private final String packageId; - private final String repo; - private final Date date; - public BuildRecord(String url, String packageId, String repo, Date date) { - super(); - this.url = url; - this.packageId = packageId; - this.repo = repo; - this.date = date; - } - public String getUrl() { - return url; - } - - public String getPackageId() { - return packageId; - } - - public String getRepo() { - return repo; - } - - public Date getDate() { - return date; - } - - } public boolean packageExists(String id, String ver) throws IOException { if (packageInstalled(id, ver)) { diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/IPackageCacheManager.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/IPackageCacheManager.java index 07bcceeea6..f1cf5768bf 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/IPackageCacheManager.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/IPackageCacheManager.java @@ -13,12 +13,27 @@ public interface IPackageCacheManager { String getPackageUrl(String packageId) throws IOException; + /** + * Load the package specified by the id and version specified by the parameters. + *

+ * Depending on the implementation logic, this may be from the package cache, or resolved from another service. + * + * @param id The package id + * @param version The package version. If this is null, the implementation logic will attempt to resolve an + * appropriate version based on the package id. + * @return the NPM Package + * @throws FHIRException + * @throws IOException + */ NpmPackage loadPackage(String id, String version) throws FHIRException, IOException; /** - * + * Load the package specified by the id and version specified in the idAndVer parameter. + *

+ * Depending on the implementation logic, this may be from the package cache, or resolved from another service. + * * @param idAndVer - use id#ver - * @return + * @return the NPM Package or * @throws FHIRException * @throws IOException */ diff --git a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java index 0708cab8ca..87f223787f 100644 --- a/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java +++ b/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/npm/NpmPackage.java @@ -671,11 +671,11 @@ public boolean isIndexed() throws IOException { } - public void checkIndexed(String desc) throws IOException { + public void checkIndexed(String path) throws IOException { for (NpmPackageFolder folder : folders.values()) { JsonObject index = folder.index(); if (index == null || index.forceArray("files").size() == 0) { - indexFolder(desc, folder); + indexFolder(path, folder); } } } @@ -687,18 +687,18 @@ public void checkIndexed(String desc) throws IOException { * See the FHIR specification for details on .index.json * format and usage. * - * @param desc + * @param path * @param folder * @throws FileNotFoundException * @throws IOException */ - public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException { + public void indexFolder(String path, NpmPackageFolder folder) throws FileNotFoundException, IOException { List remove = new ArrayList<>(); NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null); - for (String n : folder.listFiles()) { - if (!indexer.seeFile(n, folder.fetchFile(n))) { - remove.add(n); + for (String file : folder.listFiles()) { + if (!indexer.seeFile(file, folder.fetchFile(file))) { + remove.add(file); } } for (String n : remove) { @@ -715,7 +715,7 @@ public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoun } } catch (Exception e) { TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json")); - throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e); + throw new IOException("Error parsing "+(path == null ? "" : path+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e); } } diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/CIBuildClientTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/CIBuildClientTests.java new file mode 100644 index 0000000000..23115ce0cf --- /dev/null +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/CIBuildClientTests.java @@ -0,0 +1,158 @@ +package org.hl7.fhir.utilities.npm; + +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import org.junit.jupiter.api.Test; + +import okhttp3.mockwebserver.MockWebServer; +import org.mockito.Mockito; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doReturn; + +public class CIBuildClientTests { + + static final String DUMMY_PACKAGE = "my.dummy.package"; + + public static final String DUMMY_PACKAGE_URL = "http://example.org/my-dummy-package"; + public static final String DUMMY_PACKAGE_NAME = "DummyPackage"; + static final String dummyQas = "[\n" + + " {\n" + + " \"url\": \"" + DUMMY_PACKAGE_URL + "\",\n" + + " \"name\": \"" + DUMMY_PACKAGE_NAME + "\",\n" + + " \"title\": \"Dummy Package\",\n" + + " \"description\": \"Not a real package.\",\n" + + " \"status\": \"draft\",\n" + + " \"package-id\": \""+ DUMMY_PACKAGE +"\",\n" + + " \"ig-ver\": \"1.1.0\",\n" + + " \"date\": \"Sat, 30 Nov, 2024 00:31:05 +0000\",\n" + + " \"dateISO8601\": \"2024-11-30T00:31:05+00:00\",\n" + + " \"version\": \"4.0.1\",\n" + + " \"repo\": \"Dummy-Organisation/dummy-project/branches/main/qa.json\"\n" + + " }]"; + + public static final String DUMMY_PACKAGE_DATE = "20241227181832"; + static final String dummyManifest = "{\n" + + " \"version\" : \"1.1.0\",\n" + + " \"fhirVersion\" : [\"4.0.1\"],\n" + + " \"date\" : \"" + DUMMY_PACKAGE_DATE + "\",\n" + + " \"name\" : \""+ DUMMY_PACKAGE_NAME +"\",\n" + + " \"jurisdiction\" : \"urn:iso:std:iso:3166#CA\"\n" + + "}"; + + @Test + public void testOnlyOneQasQueryUnderRefreshThreshold() throws IOException, InterruptedException { + MockWebServer server = new MockWebServer(); + CIBuildClient ciBuildClient = getCiBuildClient(server); + + enqueueJson(server, dummyQas); + enqueueDummyTgzStream(server, "fake tgz"); + verifyInputStreamContent(ciBuildClient.loadFromCIBuild(DUMMY_PACKAGE, "main"), "fake tgz"); + + assertThat(server.getRequestCount()).isEqualTo(2); + assertThat(server.takeRequest().getPath()).startsWith("/ig/qas.json?nocache="); + assertThat(server.takeRequest().getPath()).startsWith("/ig/Dummy-Organisation/dummy-project/branches/main/package.tgz"); + + enqueueDummyTgzStream(server, "fake tgz 2"); + + verifyInputStreamContent(ciBuildClient.loadFromCIBuild(DUMMY_PACKAGE, "main"), "fake tgz 2"); + } + + @Test + public void testNewQasQueryOverRefreshThreshold() throws IOException, InterruptedException { + MockWebServer server = new MockWebServer(); + CIBuildClient ciBuildClient = getCiBuildClient(server); + + enqueueJson(server, dummyQas); + enqueueDummyTgzStream(server, "fake tgz"); + verifyInputStreamContent(ciBuildClient.loadFromCIBuild(DUMMY_PACKAGE, "main"), "fake tgz"); + + assertThat(server.getRequestCount()).isEqualTo(2); + assertThat(server.takeRequest().getPath()).startsWith("/ig/qas.json?nocache="); + assertThat(server.takeRequest().getPath()).startsWith("/ig/Dummy-Organisation/dummy-project/branches/main/package.tgz"); + + enqueueJson(server, dummyQas); + enqueueDummyTgzStream(server, "fake tgz 2"); + + Thread.sleep(1500); + verifyInputStreamContent(ciBuildClient.loadFromCIBuild(DUMMY_PACKAGE, "main"), "fake tgz 2"); + } + + @Test + public void testGetPackageId() { + MockWebServer server = new MockWebServer(); + CIBuildClient ciBuildClient = getCiBuildClient(server); + + enqueueJson(server, dummyQas); + + String packageId = ciBuildClient.getPackageId(DUMMY_PACKAGE_URL); + assertThat(packageId).isEqualTo(DUMMY_PACKAGE); + } + + @Test + public void testGetPackageUrl() { + MockWebServer server = new MockWebServer(); + CIBuildClient ciBuildClient = getCiBuildClient(server); + + enqueueJson(server, dummyQas); + + String packageUrl = ciBuildClient.getPackageUrl(DUMMY_PACKAGE); + assertThat(packageUrl).isEqualTo(DUMMY_PACKAGE_URL); + } + + @Test + public void testIsCurrentPackage() throws IOException { + MockWebServer server = new MockWebServer(); + CIBuildClient ciBuildClient = getCiBuildClient(server); + + enqueueJson(server, dummyQas); + + enqueueJson(server, dummyManifest); + + NpmPackage npmPackage = Mockito.spy(NpmPackage.empty()); + doReturn(DUMMY_PACKAGE_DATE).when(npmPackage).date(); + doReturn(DUMMY_PACKAGE_URL).when(npmPackage).url(); + assertTrue(ciBuildClient.isCurrent(DUMMY_PACKAGE, npmPackage)); + enqueueJson(server, dummyManifest); + + String wrongDate = "1092302309"; + assertThat(wrongDate).isNotEqualTo(DUMMY_PACKAGE_DATE); + doReturn(wrongDate).when(npmPackage).date(); + assertFalse(ciBuildClient.isCurrent(DUMMY_PACKAGE, npmPackage)); + } + + private static void enqueueDummyTgzStream(MockWebServer server, String fake_tgz) { + server.enqueue( + new MockResponse() + .setBody(fake_tgz) + .addHeader("Content-Type", "application/text") + .setResponseCode(200)); + } + + + private static CIBuildClient getCiBuildClient(MockWebServer server) { + HttpUrl serverRoot = server.url(""); + return new CIBuildClient(serverRoot.toString().substring(0, serverRoot.toString().length() - 1), 1000, false); + } + + + private static void enqueueJson(MockWebServer server, String qasJson) { + byte[] payload = qasJson.getBytes(StandardCharsets.UTF_8); + server.enqueue( + new MockResponse() + .setBody(new String(payload)) + .addHeader("Content-Type", "application/json") + .setResponseCode(200) + ); + } + + private static void verifyInputStreamContent(BasePackageCacheManager.InputStreamWithSrc inputStream, String expected) throws IOException { + String content = new String(inputStream.stream.readAllBytes()); + assertThat(content).isEqualTo(expected); + } +} diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java index 1221aeb66d..7d39b416a2 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/FilesystemPackageManagerTests.java @@ -50,8 +50,6 @@ public class FilesystemPackageManagerTests { new PackageServer(DUMMY_URL_4) ); - - @Test public void testDefaultServers() throws IOException { FilesystemPackageCacheManager filesystemPackageCacheManager = getFilesystemPackageCacheManager(false); @@ -339,7 +337,18 @@ private void assertInitializedTestCacheIsValid(File cacheDirectory, boolean dumm // Check that only packages.ini is in the cache. assertThat(files).hasSize(1); } + } + + @Test + public void generatesIndexWhenMissing() throws IOException { + String pcmPath = ManagedFileAccess.fromPath(Files.createTempDirectory("fpcm-multithreadingTest")).getAbsolutePath(); + + final FilesystemPackageCacheManager pcm = new FilesystemPackageCacheManager.Builder().withCacheFolder(pcmPath).build(); + + Assertions.assertTrue(pcm.listPackages().isEmpty()); + NpmPackage npmPackage = pcm.addPackageToCache("example.fhir.uv.myig", "1.2.3", this.getClass().getResourceAsStream("/npm/dummy-package-no-index.tgz"), "https://packages.fhir.org/example.fhir.uv.myig/1.2.3"); + assertThat(npmPackage.isIndexed()); } diff --git a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java index d8f054f3d7..2c0cba398a 100644 --- a/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java +++ b/org.hl7.fhir.utilities/src/test/java/org/hl7/fhir/utilities/npm/LockfileTestProcessUtility.java @@ -105,7 +105,7 @@ private static void lockWaitAndDelete(String path, String lockFileName, int seco System.out.println("File "+lockFileName+" is locked. Waiting for " + seconds + " seconds to release. "); Thread.sleep(seconds * 1000L); - lockFile.renameTo(ManagedFileAccess.file(File.createTempFile(lockFile.getName(), ".lock-renamed").getAbsolutePath())); + lockFile.renameTo(File.createTempFile(lockFile.getName(), ".lock-renamed", lockFile.getParentFile())); fileLock.release(); channel.close(); diff --git a/org.hl7.fhir.utilities/src/test/resources/npm/dummy-package-no-index.tgz b/org.hl7.fhir.utilities/src/test/resources/npm/dummy-package-no-index.tgz new file mode 100644 index 0000000000..7e3bc9f3d7 Binary files /dev/null and b/org.hl7.fhir.utilities/src/test/resources/npm/dummy-package-no-index.tgz differ