diff --git a/jena-base/src/main/java/org/apache/jena/atlas/net/Host.java b/jena-base/src/main/java/org/apache/jena/atlas/net/Host.java index 1028472c30c..12449dbd711 100644 --- a/jena-base/src/main/java/org/apache/jena/atlas/net/Host.java +++ b/jena-base/src/main/java/org/apache/jena/atlas/net/Host.java @@ -128,27 +128,4 @@ public static InetAddress getLocalHostLANAddress() { throw unknownHostException; } } - - // @formatter:off -// public static void main(String ... arg) throws UnknownHostException { -// try { -// for (Enumeration ifaces = NetworkInterface.getNetworkInterfaces(); ifaces.hasMoreElements();) { -// NetworkInterface iface = ifaces.nextElement(); -// // Iterate all IP addresses assigned to each card... -// for (Enumeration inetAddrs = iface.getInetAddresses(); inetAddrs.hasMoreElements();) { -// InetAddress inetAddr = inetAddrs.nextElement(); -// System.out.println("IP Address : '" +inetAddr.getHostAddress()+"'"); -// } -// } -// System.out.println(); -// -// InetAddress inetAddr = getLocalHostLANAddress(); -// //InetAddress localhost = InetAddress.getLocalHost(); -// System.out.println("System IP Address : '" +inetAddr.getHostAddress()+"'"); -// } catch (Exception ex) { -// ex.printStackTrace(); -// System.exit(0); -// } -// } - // @formatter:on } diff --git a/jena-fuseki2/jena-fuseki-main/pom.xml b/jena-fuseki2/jena-fuseki-main/pom.xml index e620084863a..b72e8613279 100644 --- a/jena-fuseki2/jena-fuseki-main/pom.xml +++ b/jena-fuseki2/jena-fuseki-main/pom.xml @@ -83,6 +83,19 @@ jetty-xml + + + org.apache.shiro + shiro-core + + + + org.apache.shiro + shiro-web + jakarta + + + org.junit.vintage junit-vintage-engine @@ -96,41 +109,39 @@ - org.junit.platform - junit-platform-suite + org.junit.jupiter + junit-jupiter-params test - + - org.apache.logging.log4j - log4j-slf4j2-impl - true - compile + org.junit.platform + junit-platform-suite + test - - - - org.apache.shiro - shiro-core - jakarta + + - org.apache.shiro - shiro-config-core + org.awaitility + awaitility test - true - org.apache.shiro - shiro-web - jakarta - test + org.apache.logging.log4j + log4j-slf4j2-impl true + compile diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/AuthorizationFilter403.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/AuthorizationFilter403.java new file mode 100644 index 00000000000..715331d1f3d --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/AuthorizationFilter403.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.authz; + +import java.io.IOException; + +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.jena.web.HttpSC; +import org.apache.shiro.web.filter.authz.AuthorizationFilter; +import org.apache.shiro.web.util.WebUtils; + +/** Specialise AuthorizationFilter to yield HTTP 403 on access denied */ +public abstract class AuthorizationFilter403 extends AuthorizationFilter +{ + private String message; + + protected AuthorizationFilter403(String text) { setMessage(text); } + protected AuthorizationFilter403() { this(null); } + + /** Set the message used in HTTP 403 responses */ + public void setMessage(String msg) { message = msg; } + + public String getMessage() { return message; } + + @Override + protected boolean onAccessDenied(ServletRequest request, ServletResponse response) throws IOException { + HttpServletResponse httpResponse; + try { httpResponse = WebUtils.toHttp(response); } + catch (ClassCastException ex) { + // Not a HTTP Servlet operation + return super.onAccessDenied(request, response); + } + if ( message == null ) + httpResponse.sendError(HttpSC.FORBIDDEN_403); + else + httpResponse.sendError(HttpSC.FORBIDDEN_403, message); + return false; // No further processing. + } +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/DenyFilter.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/DenyFilter.java new file mode 100644 index 00000000000..9e7a5d8e421 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/DenyFilter.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.authz; + +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; + +/** An authorization filter that always denies access and sends back HTTP 403 */ +public class DenyFilter extends AuthorizationFilter403 { + + public DenyFilter() { super("Access denied"); } + + @Override + protected boolean isAccessAllowed(ServletRequest request, ServletResponse response, Object mappedValue) { + return false; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/LocalhostFilter.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/LocalhostFilter.java new file mode 100644 index 00000000000..f029b56fff1 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/authz/LocalhostFilter.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.authz; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Set; + +import jakarta.servlet.ServletRequest; +import jakarta.servlet.ServletResponse; +import org.apache.shiro.web.filter.authz.PortFilter; + +/** + * A Filter that can allow or deny access based on whether the + * the host that sent the request is the loopback address (AKA localhost). + * Use of the external IP address of the local machine does not permit access, + * only the loopback interface is authorized. + * Responds with HTTP 403 on any denied request. + * + * Example: + * 
+ * [main]
+ * localhost=org.apache.jena.fuseki.authz.LocalhostFilter
+ * ...
+ * [urls]
+ * /LocalFilesForLocalPeople/** = localhost
+ *
+ * @see PortFilter + */ + +public class LocalhostFilter extends AuthorizationFilter403 { + + private static final String message = "Access denied : only localhost access allowed"; + + public LocalhostFilter() { super(message); } + + private static String LOCALHOST_IpV6_a = "[0:0:0:0:0:0:0:1]"; + private static String LOCALHOST_IpV6_b = "0:0:0:0:0:0:0:1"; + // This is what appears in the Chrome developer tools client-side. + // "[0:0:0:0:0:0:0:1]" by the time it arrives here, It is not clear which + // software component is responsible for that. + // To be safe we add "[::1]". + private static String LOCALHOST_IpV6_c = "[::1]"; + private static String LOCALHOST_IpV4 = "127.0.0.1"; // Strictly, 127.*.*.* + + private static final Collection localhosts = Set.copyOf( + Arrays.asList(LOCALHOST_IpV4, LOCALHOST_IpV6_a, LOCALHOST_IpV6_b, LOCALHOST_IpV6_c)); + + @Override + protected boolean isAccessAllowed(ServletRequest request, ServletResponse response, Object mappedValue) throws Exception { + return localhosts.contains(request.getRemoteAddr()); + } +} + + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/main/cmds/FusekiServerCmd.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/main/cmds/FusekiServerCmd.java new file mode 100644 index 00000000000..2c61c0ed09b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/main/cmds/FusekiServerCmd.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.main.cmds; + +import org.apache.jena.fuseki.run.FusekiModServer; +import org.apache.jena.fuseki.system.FusekiLogging; + +/** Fuseki command that runs a Fuseki server with the admin UI. + *

+ * Use {@code --conf=} for multiple datasets and specific service names. + *

+ * The command line dataset setup only supports a single dataset. + */ + +public class FusekiServerCmd { + // This class wraps FusekiMain so that it can take control of logging setup. + // This class does not depend via inheritance on any Jena code + // and does not trigger Jena initialization. + // FusekiLogging runs before any Jena code can trigger logging setup. + // + // Inheritance causes initialization in the super class first, before class + // initialization code in this class. + + static { FusekiLogging.setLogging(); } + + /** + * Build and run, a server based on command line syntax. This operation does not + * return. See {@link FusekiMain#build} to build a server using command line + * syntax but not start it. + */ + static public void main(String... args) { + // Fix up args + // --empty + // --modules=true + FusekiModServer.runAsync(args).join(); + } +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackup.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackup.java new file mode 100644 index 00000000000..758af27d8ac --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackup.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import static java.lang.String.format; + +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.ctl.ActionAsyncTask; +import org.apache.jena.fuseki.ctl.TaskBase; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.slf4j.Logger; + +public class ActionBackup extends ActionAsyncTask +{ + public ActionBackup() { super("Backup"); } + + @Override + public void validate(HttpAction action) {} + + @Override + protected Runnable createRunnable(HttpAction action) { + String name = getItemName(action); + if ( name == null ) { + action.log.error("Null for dataset name in item request"); + ServletOps.errorOccurred("Null for dataset name in item request"); + return null; + } + + action.log.info(format("[%d] Backup dataset %s", action.id, name)); + // ** Error changing in TaskBase + BackupTask task = new BackupTask(action); + if ( task.dataset == null ) { + ServletOps.errorBadRequest("Dataset not found"); + return null; + } + return task; + } + + static class BackupTask extends TaskBase { + static private Logger log = Fuseki.backupLog; + + public BackupTask(HttpAction action) { + super(action); + } + + @Override + public void run() { + try { + String backupFilename = Backup.chooseFileName(datasetName); + log.info(format("[%d] >>>> Start backup %s -> %s", actionId, datasetName, backupFilename)); + Backup.backup(transactional, dataset, backupFilename); + log.info(format("[%d] <<<< Finish backup %s -> %s", actionId, datasetName, backupFilename)); + } catch (Throwable ex) { + log.warn(format("[%d] **** Exception in backup", actionId), ex); + // Pass on - the async task tracking infrastructure will record this. + throw ex; + } + } + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackupList.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackupList.java new file mode 100644 index 00000000000..1e52a65b696 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionBackupList.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import static java.lang.String.format; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import org.apache.jena.atlas.json.JsonBuilder; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; + +/** + * A JSON API to list all the backups in the backup directory + */ +public class ActionBackupList extends ActionCtl { + + @Override + public void execGet(HttpAction action) { + executeLifecycle(action); + } + + @Override + public void execPost(HttpAction action) { + executeLifecycle(action); + } + + @Override + public void validate(HttpAction action) {} + + @Override + public void execute(HttpAction action) { + JsonValue result = description(action); + ServletOps.setNoCache(action); + ServletOps.sendJsonReponse(action, result); + } + + private static DirectoryStream.Filter filterVisibleFiles = (entry) -> { + File f = entry.toFile(); + return f.isFile() && !f.isHidden(); + }; + + private JsonValue description(HttpAction action) { + if ( ! Files.isDirectory(FusekiApp.dirBackups) ) + ServletOps.errorOccurred(format("[%d] Backup area '%s' is not a directory", action.id, FusekiApp.dirBackups)); + + List paths = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(FusekiApp.dirBackups, filterVisibleFiles)) { + stream.forEach(paths::add); + } catch (IOException ex) { + action.log.error(format("[%d] Backup file list :: IOException :: %s", action.id, ex.getMessage())); + ServletOps.errorOccurred(ex); + } + + List fileNames = paths.stream().map((p)->p.getFileName().toString()).sorted().collect(Collectors.toList()); + + JsonBuilder builder = new JsonBuilder(); + builder.startObject("top"); + builder.key("backups"); + + builder.startArray(); + fileNames.forEach(builder::value); + builder.finishArray(); + + builder.finishObject("top"); + return builder.build(); + + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionCompact.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionCompact.java new file mode 100644 index 00000000000..3639db5267c --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionCompact.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +public class ActionCompact { + +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionDatasets.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionDatasets.java new file mode 100644 index 00000000000..7630b8c6f02 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionDatasets.java @@ -0,0 +1,499 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import static java.lang.String.format; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +import jakarta.servlet.http.HttpServletRequest; +import org.apache.commons.lang3.StringUtils; +import org.apache.jena.atlas.RuntimeIOException; +import org.apache.jena.atlas.io.IO; +import org.apache.jena.atlas.json.JsonBuilder; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.atlas.lib.FileOps; +import org.apache.jena.atlas.lib.InternalErrorException; +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.atlas.web.ContentType; +import org.apache.jena.datatypes.xsd.XSDDatatype; +import org.apache.jena.fuseki.build.DatasetDescriptionMap; +import org.apache.jena.fuseki.build.FusekiConfig; +import org.apache.jena.fuseki.ctl.ActionContainerItem; +import org.apache.jena.fuseki.ctl.JsonDescription; +import org.apache.jena.fuseki.server.DataAccessPoint; +import org.apache.jena.fuseki.server.DataService; +import org.apache.jena.fuseki.server.FusekiVocab; +import org.apache.jena.fuseki.server.ServerConst; +import org.apache.jena.fuseki.servlets.ActionLib; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.fuseki.system.DataUploader; +import org.apache.jena.fuseki.system.FusekiNetLib; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.*; +import org.apache.jena.riot.*; +import org.apache.jena.riot.system.StreamRDF; +import org.apache.jena.riot.system.StreamRDFLib; +import org.apache.jena.shared.JenaException; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.Quad; +import org.apache.jena.sparql.core.assembler.AssemblerUtils; +import org.apache.jena.sparql.util.FmtUtils; +import org.apache.jena.vocabulary.RDF; +import org.apache.jena.web.HttpSC; + +public class ActionDatasets extends ActionContainerItem { + + + static private Property pServiceName = FusekiVocab.pServiceName; + //static private Property pStatus = FusekiVocab.pStatus; + + private static final String paramDatasetName = "dbName"; + private static final String paramDatasetType = "dbType"; + private static final String tDatabaseTDB = "tdb"; + private static final String tDatabaseTDB2 = "tdb2"; + private static final String tDatabaseMem = "mem"; + + public ActionDatasets() { super(); } + + @Override + public void validate(HttpAction action) { } + + // ---- GET : return details of dataset or datasets. + @Override + protected JsonValue execGetContainer(HttpAction action) { + action.log.info(format("[%d] GET datasets", action.id)); + JsonBuilder builder = new JsonBuilder(); + builder.startObject("D"); + builder.key(ServerConst.datasets); + JsonDescription.arrayDatasets(builder, action.getDataAccessPointRegistry()); + builder.finishObject("D"); + return builder.build(); + } + + @Override + protected JsonValue execGetItem(HttpAction action) { + String item = getItemDatasetName(action); + action.log.info(format("[%d] GET dataset %s", action.id, item)); + JsonBuilder builder = new JsonBuilder(); + DataAccessPoint dsDesc = getItemDataAccessPoint(action, item); + if ( dsDesc == null ) + ServletOps.errorNotFound("Not found: dataset "+item); + JsonDescription.describe(builder, dsDesc); + return builder.build(); + } + + // ---- POST + + @Override + protected JsonValue execPostContainer(HttpAction action) { + UUID uuid = UUID.randomUUID(); + + ContentType ct = ActionLib.getContentType(action); + + boolean hasParams = action.getRequestParameterNames().hasMoreElements(); + + if ( ct == null && ! hasParams ) + ServletOps.errorBadRequest("Bad request - Content-Type or both parameters dbName and dbType required"); + + boolean succeeded = false; + String systemFileCopy = null; + String configFile = null; + + DatasetDescriptionMap registry = new DatasetDescriptionMap(); + + synchronized (FusekiAdmin.systemLock) { + try { + // Where to build the templated service/database. + Model descriptionModel = ModelFactory.createDefaultModel(); + StreamRDF dest = StreamRDFLib.graph(descriptionModel.getGraph()); + + if ( hasParams || WebContent.isHtmlForm(ct) ) + assemblerFromForm(action, dest); + else if ( WebContent.isMultiPartForm(ct) ) + assemblerFromUpload(action, dest); + else + assemblerFromBody(action, dest); + + // ---- + // Keep a persistent copy immediately. This is not used for + // anything other than being "for the record". + systemFileCopy = FusekiApp.dirSystemFileArea.resolve(uuid.toString()).toString(); + try ( OutputStream outCopy = IO.openOutputFile(systemFileCopy) ) { + RDFDataMgr.write(outCopy, descriptionModel, Lang.TURTLE); + } + + // ---- + // Add the dataset and graph wiring for assemblers + Model model = ModelFactory.createDefaultModel(); + model.add(descriptionModel); + // See AssemblerUtils.readAssemblerFile(String filename) + AssemblerUtils.addRegistered(model); + + // ---- + // Process configuration. + + // Returns the "service fu:name NAME" statement + Statement stmt = findService(model); + + Resource subject = stmt.getSubject(); + Literal object = stmt.getObject().asLiteral(); + + if ( object.getDatatype() != null && ! object.getDatatype().equals(XSDDatatype.XSDstring) ) + action.log.warn(format("[%d] Service name '%s' is not a string", action.id, FmtUtils.stringForRDFNode(object))); + + String datasetPath; + { // Check the name provided. + String datasetName = object.getLexicalForm(); + // This duplicates the code FusekiBuilder.buildDataAccessPoint to give better error messages and HTTP status code." + + // ---- Check and canonicalize name. + if ( datasetName.isEmpty() ) + ServletOps.error(HttpSC.BAD_REQUEST_400, "Empty dataset name"); + if ( StringUtils.isBlank(datasetName) ) + ServletOps.error(HttpSC.BAD_REQUEST_400, format("Whitespace dataset name: '%s'", datasetName)); + if ( datasetName.contains(" ") ) + ServletOps.error(HttpSC.BAD_REQUEST_400, format("Bad dataset name (contains spaces) '%s'",datasetName)); + if ( datasetName.equals("/") ) + ServletOps.error(HttpSC.BAD_REQUEST_400, format("Bad dataset name '%s'",datasetName)); + datasetPath = DataAccessPoint.canonical(datasetName); + // ---- Check whether it already exists + if ( action.getDataAccessPointRegistry().isRegistered(datasetPath) ) + ServletOps.error(HttpSC.CONFLICT_409, "Name already registered "+datasetPath); + } + + action.log.info(format("[%d] Create database : name = %s", action.id, datasetPath)); + + configFile = FusekiApp.generateConfigurationFilename(datasetPath); + List existing = FusekiApp.existingConfigurationFile(datasetPath); + if ( ! existing.isEmpty() ) + ServletOps.error(HttpSC.CONFLICT_409, "Configuration file for '"+datasetPath+"' already exists"); + + // Write to configuration directory. + try ( OutputStream outCopy = IO.openOutputFile(configFile) ) { + RDFDataMgr.write(outCopy, descriptionModel, Lang.TURTLE); + } + + // Need to be in Resource space at this point. + DataAccessPoint dataAccessPoint = FusekiConfig.buildDataAccessPoint(subject.getModel().getGraph(), subject.asNode(), registry); + if ( dataAccessPoint == null ) { + FmtLog.error(action.log, "Failed to build DataAccessPoint: datasetPath = %s; DataAccessPoint name = %s", datasetPath, dataAccessPoint); + ServletOps.errorBadRequest("Failed to build DataAccessPoint"); + return null; + } + dataAccessPoint.getDataService().setEndpointProcessors(action.getOperationRegistry()); + dataAccessPoint.getDataService().goActive(); + if ( ! datasetPath.equals(dataAccessPoint.getName()) ) + FmtLog.warn(action.log, "Inconsistent names: datasetPath = %s; DataAccessPoint name = %s", datasetPath, dataAccessPoint); + succeeded = true; + + action.getDataAccessPointRegistry().register(dataAccessPoint); + action.setResponseContentType(WebContent.contentTypeTextPlain); + ServletOps.success(action); + } catch (IOException ex) { IO.exception(ex); } + finally { + if ( ! succeeded ) { + if ( systemFileCopy != null ) FileOps.deleteSilent(systemFileCopy); + if ( configFile != null ) FileOps.deleteSilent(configFile); + } + } + return null; + } + } + + /** Find the service resource. There must be only one in the configuration. */ + private Statement findService(Model model) { + // Try to find by unique pServiceName (max backwards compatibility) + // then try to find by rdf:type fuseki:Service. + + Statement stmt = getOne(model, null, pServiceName, null); + // null means 0 or many, not one. + + if ( stmt == null ) { + // This calculates { ?x rdf:type fu:Service ; ?x fu:name ?name } + // One and only one service. + Statement stmt2 = getOne(model, null, RDF.type, FusekiVocab.fusekiService); + if ( stmt2 == null ) { + int count = model.listStatements(null, RDF.type, FusekiVocab.fusekiService).toList().size(); + if ( count == 0 ) + ServletOps.errorBadRequest("No triple rdf:type fuseki:Service found"); + else + ServletOps.errorBadRequest("Multiple Fuseki service descriptions"); + } + Statement stmt3 = getOne(model, stmt2.getSubject(), pServiceName, null); + if ( stmt3 == null ) { + StmtIterator sIter = model.listStatements(stmt2.getSubject(), pServiceName, (RDFNode)null ); + if ( ! sIter.hasNext() ) + ServletOps.errorBadRequest("No name given in description of Fuseki service"); + sIter.next(); + if ( sIter.hasNext() ) + ServletOps.errorBadRequest("Multiple names given in description of Fuseki service"); + throw new InternalErrorException("Inconsistent: getOne didn't fail the second time"); + } + stmt = stmt3; + } + + if ( ! stmt.getObject().isLiteral() ) + ServletOps.errorBadRequest("Found "+FmtUtils.stringForRDFNode(stmt.getObject())+" : Service names are strings, then used to build the external URI"); + + return stmt; + } + + @Override + protected JsonValue execPostItem(HttpAction action) { + String name = getItemDatasetName(action); + if ( name == null ) + name = "''"; + action.log.info(format("[%d] POST dataset %s", action.id, name)); + + // Not in the action - this not an ActionService. + DataAccessPoint dap = getItemDataAccessPoint(action, name); + + if ( dap == null ) + ServletOps.errorNotFound("Not found: dataset "+name); + + DataService dSrv = dap.getDataService(); + if ( dSrv == null ) + // If not set explicitly, take from DataAccessPoint + dSrv = action.getDataAccessPoint().getDataService(); + + String s = action.getRequestParameter("state"); + if ( s == null || s.isEmpty() ) + ServletOps.errorBadRequest("No state change given"); + return null; + } + + // ---- DELETE + + @Override + protected void execDeleteItem(HttpAction action) { + // Does not exist? + String name = getItemDatasetName(action); + if ( name == null ) + name = ""; + action.log.info(format("[%d] DELETE dataset=%s", action.id, name)); + + if ( ! action.getDataAccessPointRegistry().isRegistered(name) ) + ServletOps.errorNotFound("No such dataset registered: "+name); + + boolean succeeded = false; + + synchronized(FusekiAdmin.systemLock ) { + try { + // Here, go offline. + // Need to reference count operations when they drop to zero + // or a timer goes off, we delete the dataset. + + // Redo check inside transaction. + DataAccessPoint ref = action.getDataAccessPointRegistry().get(name); + if ( ref == null ) + ServletOps.errorNotFound("No such dataset registered: "+name); + + // Get a reference before removing. + DataService dataService = ref.getDataService(); + // ---- Make it invisible in this running server. + action.getDataAccessPointRegistry().remove(name); + + // Find the configuration. + String filename = name.startsWith("/") ? name.substring(1) : name; + List configurationFiles = FusekiApp.existingConfigurationFile(filename); + + if ( configurationFiles.isEmpty() ) { + // ---- Unmanaged + action.log.warn(format("[%d] Can't delete database configuration - not a managed database", action.id, name)); +// ServletOps.errorOccurred(format("Can't delete database - not a managed configuration", name)); + succeeded = true; + ServletOps.success(action); + return; + } + + if ( configurationFiles.size() > 1 ) { + // -- This should not happen. + action.log.warn(format("[%d] There are %d configuration files, not one.", action.id, configurationFiles.size())); + ServletOps.errorOccurred(format("There are %d configuration files, not one. Delete not performed; manual clean up of the filesystem needed.", + configurationFiles.size())); + return; + } + + // ---- Remove managed database. + String cfgPathname = configurationFiles.get(0); + + // Delete configuration file. + // Once deleted, server restart will not have the database. + FileOps.deleteSilent(cfgPathname); + + // Delete the database for real only when it is in the server "run/databases" + // area. Don't delete databases that reside elsewhere. We do delete the + // configuration file, so the databases will not be associated with the server + // anymore. + + boolean isTDB1 = org.apache.jena.tdb1.sys.TDBInternal.isTDB1(dataService.getDataset()); + boolean isTDB2 = org.apache.jena.tdb2.sys.TDBInternal.isTDB2(dataService.getDataset()); + + // TODO This occasionally fails in tests due to outstanding transactions. + try { + dataService.shutdown(); + } catch (JenaException ex) { + return; + } + // JENA-1481: Really delete files. + if ( ( isTDB1 || isTDB2 ) ) { + // Delete databases created by the UI, or the admin operation, which are + // in predictable, unshared location on disk. + // There may not be any database files, the in-memory case. + Path pDatabase = FusekiApp.dirDatabases.resolve(filename); + if ( Files.exists(pDatabase)) { + try { + if ( Files.isSymbolicLink(pDatabase)) { + action.log.info(format("[%d] Database is a symbolic link, not removing files", action.id, pDatabase)); + } else { + IO.deleteAll(pDatabase); + action.log.info(format("[%d] Deleted database files %s", action.id, pDatabase)); + } + } catch (RuntimeIOException ex) { + action.log.error(format("[%d] Error while deleting database files %s: %s", action.id, pDatabase, ex.getMessage()), ex); + // But we have managed to remove it from the running server, and removed its configuration, so declare victory. + } + } + } + + succeeded = true; + ServletOps.success(action); + } finally { + // No clearup needed + } + } + } + + private static void assemblerFromBody(HttpAction action, StreamRDF dest) { + bodyAsGraph(action, dest); + } + + private static Map dbTypeToTemplate = new HashMap<>(); + static { + dbTypeToTemplate.put(tDatabaseTDB, Template.templateTDB1_FN); + dbTypeToTemplate.put(tDatabaseTDB2, Template.templateTDB2_FN); + dbTypeToTemplate.put(tDatabaseMem, Template.templateTIM_MemFN); + } + + private static void assemblerFromForm(HttpAction action, StreamRDF dest) { + String x = action.getRequestQueryString(); + String dbType = action.getRequestParameter(paramDatasetType); + String dbName = action.getRequestParameter(paramDatasetName); + if ( StringUtils.isBlank(dbType) || StringUtils.isBlank(dbName) ) + ServletOps.errorBadRequest("Received HTML form. Both parameters 'dbName' and 'dbType' required"); + + Map params = new HashMap<>(); + + if ( dbName.startsWith("/") ) + params.put(Template.NAME, dbName.substring(1)); + else + params.put(Template.NAME, dbName); + FusekiApp.addGlobals(params); + + //action.log.info(format("[%d] Create database : name = %s, type = %s", action.id, dbName, dbType )); + + String template = dbTypeToTemplate.get(dbType.toLowerCase(Locale.ROOT)); + if ( template == null ) { + List keys = new ArrayList<>(dbTypeToTemplate.keySet()); + Collections.sort(keys); + ServletOps.errorBadRequest(format("dbType can be only one of %s", keys)); + } + + String instance = TemplateFunctions.templateFile(template, params, Lang.TTL); + RDFParser.create().source(new StringReader(instance)).base("http://base/").lang(Lang.TTL).parse(dest); + } + + private static void assemblerFromUpload(HttpAction action, StreamRDF dest) { + DataUploader.incomingData(action, dest); + } + + // [ADMIN] +// // Persistent state change. +// private static void setDatasetState(String name, Resource newState) { +// boolean committed = false; +// system.begin(ReadWrite.WRITE); +// try { +// String dbName = name; +// if ( dbName.startsWith("/") ) +// dbName = dbName.substring(1); +// +// String update = StrUtils.strjoinNL +// (PREFIXES, +// "DELETE { GRAPH ?g { ?s fu:status ?state } }", +// "INSERT { GRAPH ?g { ?s fu:status "+FmtUtils.stringForRDFNode(newState)+" } }", +// "WHERE {", +// " GRAPH ?g { ?s fu:name '"+dbName+"'; ", +// " fu:status ?state .", +// " }", +// "}" +// ); +// UpdateRequest req = UpdateFactory.create(update); +// UpdateAction.execute(req, system); +// system.commit(); +// committed = true; +// } finally { +// if ( ! committed ) system.abort(); +// system.end(); +// } +// } + + // ---- Auxiliary functions + + private static Quad getOne(DatasetGraph dsg, Node g, Node s, Node p, Node o) { + Iterator iter = dsg.findNG(g, s, p, o); + if ( ! iter.hasNext() ) + return null; + Quad q = iter.next(); + if ( iter.hasNext() ) + return null; + return q; + } + + private static Statement getOne(Model m, Resource s, Property p, RDFNode o) { + StmtIterator iter = m.listStatements(s, p, o); + if ( ! iter.hasNext() ) + return null; + Statement stmt = iter.next(); + if ( iter.hasNext() ) + return null; + return stmt; + } + + // TODO Merge with Upload.incomingData + + private static void bodyAsGraph(HttpAction action, StreamRDF dest) { + HttpServletRequest request = action.getRequest(); + String base = ActionLib.wholeRequestURL(request); + ContentType ct = FusekiNetLib.getContentType(request); + Lang lang = RDFLanguages.contentTypeToLang(ct.getContentTypeStr()); + if ( lang == null ) { + ServletOps.errorBadRequest("Unknown content type for triples: " + ct); + return; + } + dest.prefix("root", base+"#"); + ActionLib.parseOrError(action, dest, lang, base); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionLogs.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionLogs.java new file mode 100644 index 00000000000..3b91abf0c47 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionLogs.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import static org.apache.jena.riot.WebContent.charsetUTF8; +import static org.apache.jena.riot.WebContent.contentTypeTextPlain; + +import java.io.IOException; + +import jakarta.servlet.ServletOutputStream; +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; + +public class ActionLogs extends ActionCtl +{ + public ActionLogs() { super(); } + + @Override + public void validate(HttpAction action) {} + + @Override + public void execGet(HttpAction action) { + executeLifecycle(action); + } + + @Override + public void execute(HttpAction action) { + try { + ServletOutputStream out = action.getResponseOutputStream(); + action.setResponseContentType(contentTypeTextPlain); + action.setResponseCharacterEncoding(charsetUTF8); + out.println("Not implemented yet"); + out.println(); + out.flush(); + ServletOps.success(action); + } catch (IOException ex) { ServletOps.errorOccurred(ex); } + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionReload.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionReload.java new file mode 100644 index 00000000000..b85fddbbd99 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ActionReload.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.main.FusekiLib; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.riot.web.HttpNames; + +/** + * Administration action to reload the server's dataset configuration. + *

+ * This is done by reading the configuration file which may have changed since server startup. + *

+ * If the server does not have a configuration file (e.g. command line or programmatic configuration) + */ +public class ActionReload extends ActionCtl { + + @Override + public void validate(HttpAction action) { + if ( action.getRequestMethod() != HttpNames.METHOD_POST ) { + ServletOps.errorMethodNotAllowed(action.getRequestMethod()); + } + } + + @Override + public void execute(HttpAction action) { + FusekiServer server = FusekiServer.get(action.getRequest().getServletContext()); + if ( server == null ) { + ServletOps.errorOccurred("Failed to find the server for this action"); + return; + } + + String configFilename = server.getConfigFilename(); + if ( configFilename == null ) { + FmtLog.warn(Fuseki.serverLog, "[%d] Server does not have an associated configuration file", action.id); + ServletOps.errorBadRequest("Server does not have an associated configuration file"); + return; + } + Model model = RDFParser.source(configFilename).toModel(); + FmtLog.info(Fuseki.serverLog, "[%d] Reload configuration", action.id); + FusekiLib.reload(server, model); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Backup.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Backup.java new file mode 100644 index 00000000000..70ea5f51563 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Backup.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import java.io.BufferedOutputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.zip.GZIPOutputStream; + +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.atlas.lib.DateTimeUtils; +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.FusekiException; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFDataMgr; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.Transactional; +import org.apache.jena.sparql.core.TransactionalNull; +import org.apache.jena.system.Txn; + +/** Perform a backup */ +public class Backup +{ + public static String chooseFileName(String dsName) { + // Without the "/" - i.e. a relative name. + String ds = dsName; + if ( ds.startsWith("/") ) + ds = ds.substring(1); + if ( ds.contains("/") ) { + Fuseki.adminLog.warn("Dataset name: weird format: "+dsName); + // Some kind of fixup + ds = ds.replace("/", "_"); + } + + String timestamp = DateTimeUtils.nowAsString("yyyy-MM-dd_HH-mm-ss"); + String filename = ds + "_" + timestamp; + filename = FusekiApp.dirBackups.resolve(filename).toString(); + return filename; + } + + // Record of all backups so we don't attempt to backup the + // same dataset multiple times at the same time. + private static Set activeBackups = Collections.newSetFromMap(new ConcurrentHashMap<>()); + + /** + * Perform a backup. + *

+ * A backup is a dump of the dataset in compressed N-Quads, done inside a transaction. + */ + public static void backup(Transactional transactional, DatasetGraph dsg, String backupfile) { + if ( transactional == null ) + transactional = new TransactionalNull(); + Txn.executeRead(transactional, ()->backup(dsg, backupfile)); + } + + // This seems to achieve about the same as "gzip -6" + // It's not too expensive in elapsed time but it's not + // zero cost. GZip, large buffer. + private static final boolean USE_GZIP = true; + + /** + * Perform a backup. + * + * @see #backup(Transactional, DatasetGraph, String) + */ + + private static void backup(DatasetGraph dsg, String backupfile) { + if (dsg == null) { + throw new FusekiException("No dataset provided to backup"); + } + + // Per backup source lock. + synchronized(activeBackups) { + // Atomically check-and-set + if ( activeBackups.contains(dsg) ) + FmtLog.warn(Fuseki.serverLog, "Backup already in progress"); + activeBackups.add(dsg); + } + + if ( !backupfile.endsWith(".nq") ) + backupfile = backupfile + ".nq"; + + if ( USE_GZIP ) + backupfile = backupfile + ".gz"; + + try { + IOX.safeWrite(Path.of(backupfile), outfile -> { + OutputStream out = outfile; + if ( USE_GZIP ) + out = new GZIPOutputStream(outfile, 8 * 1024); + try (OutputStream out2 = new BufferedOutputStream(out)) { + RDFDataMgr.write(out2, dsg, Lang.NQUADS); + } + }); + } finally { + // Remove lock. + synchronized(activeBackups) { + activeBackups.remove(dsg); + } + } + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiAdmin.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiAdmin.java new file mode 100644 index 00000000000..2481c4f6996 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiAdmin.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +public class FusekiAdmin { + public final static Object systemLock = new Object(); +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiApp.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiApp.java new file mode 100644 index 00000000000..d6cd0dc5708 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/FusekiApp.java @@ -0,0 +1,425 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import static java.lang.String.format; +import static org.apache.jena.atlas.lib.Lib.getenv; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; + +import org.apache.jena.atlas.RuntimeIOException; +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.atlas.lib.FileOps; +import org.apache.jena.atlas.lib.InternalErrorException; +import org.apache.jena.atlas.lib.Lib; +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.cmd.CmdException; +import org.apache.jena.dboe.sys.Names; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.fuseki.build.DatasetDescriptionMap; +import org.apache.jena.fuseki.build.FusekiConfig; +import org.apache.jena.fuseki.server.DataAccessPoint; +import org.apache.jena.fuseki.server.DataService; +import org.apache.jena.fuseki.server.FusekiVocabG; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Node; +import org.apache.jena.rdf.model.*; +import org.apache.jena.riot.Lang; +import org.apache.jena.riot.RDFLanguages; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.assembler.AssemblerUtils; +import org.apache.jena.system.G; + +public class FusekiApp { + // Relative names of directories in the FUSEKI_BASE area. + public static final String databasesLocationBase = "databases"; + // Place to put Lucene text and spatial indexes. + //private static final String databaseIndexesDir = "indexes"; + + public static final String backupDirNameBase = "backups"; + public static final String configDirNameBase = "configuration"; + public static final String logsNameBase = "logs"; + public static final String systemFileAreaBase = "system_files"; + public static final String templatesNameBase = "templates"; + public static final String DFT_SHIRO_INI = "shiro.ini"; + public static final String DFT_CONFIG = "config.ttl"; + + private static int BaseFusekiAutoModuleLevel = 500; + public static int levelFModAdmin = BaseFusekiAutoModuleLevel; + public static int levelFModUI = BaseFusekiAutoModuleLevel+10; + public static int levelFModShiro = BaseFusekiAutoModuleLevel+20; + + + /** Directory for TDB databases - this is known to the assembler templates */ + public static Path dirDatabases = null; + + /** Directory for writing backups */ + public static Path dirBackups = null; + + /** Directory for assembler files */ + public static Path dirConfiguration = null; + + /** Directory for assembler files */ + public static Path dirLogs = null; + +// /** Directory for system database */ +// public static Path dirSystemDatabase = null; + + /** Directory for files uploaded (e.g upload assembler descriptions); not data uploads. */ + public static Path dirSystemFileArea = null; + + /** Directory for assembler files */ + public static Path dirTemplates = null; + + private static boolean initialized = false; + // Marks the end of successful initialization. + /*package*/static boolean serverInitialized = false; + + + +// /** +// * Root of the Fuseki installation for fixed files. +// * This may be null (e.g. running inside a web application container) +// */ +// public static Path FUSEKI_HOME = null; + + /** + * Root of the varying files in this deployment. Often $PWD/run. + * This must be writable. + */ + public static Path FUSEKI_BASE = set_FUSEKI_BASE(); + + public static String envFusekiBase = "FUSEKI_BASE"; + public static String envFusekiShiro = "FUSEKI_SHIRO"; + + private static Path set_FUSEKI_BASE() { + // Does not guarantee existence + Path setting = null; + if ( FUSEKI_BASE == null ) + setting = calc_FUSEKI_BASE(); + setting = setting.toAbsolutePath(); + return setting; + } + + private static Path calc_FUSEKI_BASE() { + String valueFusekiBase = getenv("FUSEKI_BASE"); + if ( valueFusekiBase == null ) + valueFusekiBase = dftFusekiBase; + return Path.of(valueFusekiBase); + } + + // Default - "run" in the current directory. + public static final String dftFusekiBase = "run"; + + static void setEnvironment() { + if ( FUSEKI_BASE == null ) + FUSEKI_BASE = set_FUSEKI_BASE(); + + FmtLog.info(Fuseki.configLog, "FUSEKI_BASE=%s", FUSEKI_BASE); + if ( ! Files.exists(FUSEKI_BASE) ) { + try { + Files.createDirectories(FUSEKI_BASE); + } catch (IOException e) { + throw new FusekiConfigException("Failed to create FUSEKI_BASE: "+FUSEKI_BASE); + } + } + // Further checks in ensureBaseArea + } + + public static Path setup() { + // Command line arguments "--base" ... + setEnvironment(); + // Format the BASE area. + FusekiApp.ensureBaseArea(FUSEKI_BASE); + return FUSEKI_BASE; + } + + /** + * Create directories if found to be missing. + */ + public static void ensureBaseArea(Path FUSEKI_BASE) { + if ( Files.exists(FUSEKI_BASE) ) { + if ( ! Files.isDirectory(FUSEKI_BASE) ) + throw new FusekiConfigException("FUSEKI_BASE is not a directory: "+FUSEKI_BASE); + if ( ! Files.isWritable(FUSEKI_BASE) ) + throw new FusekiConfigException("FUSEKI_BASE is not writable: "+FUSEKI_BASE); + } else { + ensureDir(FUSEKI_BASE); + } + + // Ensure FUSEKI_BASE has the assumed directories. + dirTemplates = writeableDirectory(FUSEKI_BASE, templatesNameBase); + dirDatabases = writeableDirectory(FUSEKI_BASE, databasesLocationBase); + dirBackups = writeableDirectory(FUSEKI_BASE, backupDirNameBase); + dirConfiguration = writeableDirectory(FUSEKI_BASE, configDirNameBase); + dirLogs = writeableDirectory(FUSEKI_BASE, logsNameBase); + dirSystemFileArea = writeableDirectory(FUSEKI_BASE, systemFileAreaBase); + + // ---- Initialize with files. + +// // Copy missing files into FUSEKI_BASE + // Interacts with FMod_Shiro. + if ( Lib.getenv(FusekiApp.envFusekiShiro) == null ) { + copyFileIfMissing(null, DFT_SHIRO_INI, FUSEKI_BASE); + System.setProperty(FusekiApp.envFusekiShiro, FUSEKI_BASE.resolve(DFT_SHIRO_INI).toString()); + } + + copyFileIfMissing(null, DFT_CONFIG, FUSEKI_BASE); + for ( String n : Template.templateNames ) { + copyFileIfMissing(null, n, FUSEKI_BASE); + } + + serverInitialized = true; + } + + /** Copy a file from src to dst under name fn. + * If src is null, try as a classpath resource + * @param src Source directory, or null meaning use java resource. + * @param fn File name, a relative path. + * @param dst Destination directory. + * + */ + private static void copyFileIfMissing(Path src, String fn, Path dst) { + // fn may be a path. + Path dstFile = dst.resolve(fn); + if ( Files.exists(dstFile) ) + return; + if ( src != null ) { + Path srcFile = src.resolve(fn); + if ( ! Files.exists(dstFile) ) + throw new FusekiConfigException("File not found: "+srcFile); + try { + IOX.safeWrite(dstFile, output->Files.copy(srcFile, output)); + } catch (RuntimeIOException e) { + throw new FusekiConfigException("Failed to copy file "+srcFile+" to "+dstFile, e); + } + } else { + copyFileFromResource(fn, dstFile); + } + } + + private static void copyFileFromResource(String fn, Path dstFile) { + try { + // Get from the file from area "org/apache/jena/fuseki/server" + String absName = "org/apache/jena/fuseki/server/"+fn; + InputStream input = FusekiApp.class + // Else prepends classname as path + .getClassLoader() + .getResourceAsStream(absName); + + if ( input == null ) + throw new FusekiConfigException("Failed to find resource '"+absName+"'"); + IOX.safeWrite(dstFile, (output)-> input.transferTo(output)); + } + catch (RuntimeException e) { + throw new FusekiConfigException("Failed to copy "+fn+" to "+dstFile, e); + } + } + + private static List processServerConfigFile(String configFilename) { + if ( ! FileOps.exists(configFilename) ) { + Fuseki.configLog.warn("Configuration file '" + configFilename+"' does not exist"); + return Collections.emptyList(); + } + //Fuseki.configLog.info("Configuration file: " + configFilename); + Model model = AssemblerUtils.readAssemblerFile(configFilename); + if ( model.size() == 0 ) + return Collections.emptyList(); + List x = FusekiConfig.processServerConfiguration(model, Fuseki.getContext()); + return x; + } + + private static DataAccessPoint configFromTemplate(String templateFile, String datasetPath, + boolean allowUpdate, Map params) { + // ---- Setup + if ( params == null ) { + params = new HashMap<>(); + params.put(Template.NAME, datasetPath); + } else { + if ( ! params.containsKey(Template.NAME) ) { + Fuseki.configLog.warn("No NAME found in template parameters (added)"); + params.put(Template.NAME, datasetPath); + } + } + //-- Logging + Fuseki.configLog.info("Template file: " + templateFile); + String dir = params.get(Template.DIR); + if ( dir != null ) { + if ( ! Objects.equals(dir, Names.memName) && !FileOps.exists(dir) ) + throw new CmdException("Directory not found: " + dir); + } + //-- Logging + + datasetPath = DataAccessPoint.canonical(datasetPath); + + // DRY -- ActionDatasets (and others?) + addGlobals(params); + + String str = TemplateFunctions.templateFile(templateFile, params, Lang.TTL); + Lang lang = RDFLanguages.filenameToLang(str, Lang.TTL); + + Graph configuration = RDFParser.fromString(str, lang).toGraph(); + List x = G.listPO(configuration, FusekiVocabG.pServiceName, null); + if ( x.isEmpty() ) + ServletOps.errorBadRequest("No name given in description of Fuseki service"); + if ( x.size() > 1 ) + ServletOps.errorBadRequest("Multiple names given in description of Fuseki service"); + Node fusekiService = x.get(0); + DatasetDescriptionMap registry = new DatasetDescriptionMap(); + DataAccessPoint dap = FusekiConfig.buildDataAccessPoint(configuration, fusekiService, registry); + return dap; + } + + public static void addGlobals(Map params) { + if ( params == null ) { + Fuseki.configLog.warn("FusekiApp.addGlobals : params is null", new Throwable()); + return; + } + + if ( ! params.containsKey("FUSEKI_BASE") ) + params.put("FUSEKI_BASE", pathStringOrElse(FUSEKI_BASE, "unset")); +// if ( ! params.containsKey("FUSEKI_HOME") ) +// params.put("FUSEKI_HOME", pathStringOrElse(FusekiAppEnv.FUSEKI_HOME, "unset")); + } + + private static String pathStringOrElse(Path path, String dft) { + if ( path == null ) + return dft; + return path.toString(); + } + + // DRY -- ActionDatasets (and others?) + private static Statement getOne(Model m, Resource s, Property p, RDFNode o) { + StmtIterator iter = m.listStatements(s, p, o); + if ( ! iter.hasNext() ) + return null; + Statement stmt = iter.next(); + if ( iter.hasNext() ) + return null; + return stmt; + } + + private static DataAccessPoint datasetDefaultConfiguration(String name, DatasetGraph dsg, boolean allowUpdate) { + name = DataAccessPoint.canonical(name); + DataService ds = FusekiConfig.buildDataServiceStd(dsg, allowUpdate); + DataAccessPoint dap = new DataAccessPoint(name, ds); + return dap; + } + + // ---- Helpers + + /** Ensure a directory exists, creating it if necessary. + */ + private static void ensureDir(Path directory) { + File dir = directory.toFile(); + if ( ! dir.exists() ) { + boolean b = dir.mkdirs(); + if ( ! b ) + throw new FusekiConfigException("Failed to create directory: "+directory); + } + else if ( ! dir.isDirectory()) + throw new FusekiConfigException("Not a directory: "+directory); + } + + private static void mustExist(Path directory) { + File dir = directory.toFile(); + if ( ! dir.exists() ) + throw new FusekiConfigException("Does not exist: "+directory); + if ( ! dir.isDirectory()) + throw new FusekiConfigException("Not a directory: "+directory); + } + + private static boolean emptyDir(Path dir) { + return dir.toFile().list().length <= 2; + } + + private static boolean exists(Path directory) { + File dir = directory.toFile(); + return dir.exists(); + } + + private static Path writeableDirectory(Path root , String relName ) { + Path p = makePath(root, relName); + ensureDir(p); + if ( ! Files.isWritable(p) ) + throw new FusekiConfigException("Not writable: "+p); + return p; + } + + private static Path makePath(Path root , String relName ) { + Path path = root.resolve(relName); + // Must exist +// try { path = path.toRealPath(); } +// catch (IOException e) { IO.exception(e); } + return path; + } + + /** + * Dataset set name to configuration file name. Return a configuration file name - + * existing one or ".ttl" form if new + */ + public static String datasetNameToConfigurationFile(HttpAction action, String dsName) { + List existing = existingConfigurationFile(dsName); + if ( ! existing.isEmpty() ) { + if ( existing.size() > 1 ) { + action.log.warn(format("[%d] Multiple existing configuration files for %s : %s", + action.id, dsName, existing)); + ServletOps.errorBadRequest("Multiple existing configuration files for "+dsName); + return null; + } + return existing.get(0).toString(); + } + + return generateConfigurationFilename(dsName); + } + + /** New configuration file name - absolute filename */ + public static String generateConfigurationFilename(String dsName) { + String filename = dsName; + // Without "/" + if ( filename.startsWith("/")) + filename = filename.substring(1); + Path p = FusekiApp.dirConfiguration.resolve(filename+".ttl"); + return p.toString(); + } + + /** Return the filenames of all matching files in the configuration directory (absolute paths returned ). */ + public static List existingConfigurationFile(String baseFilename) { + try { + List paths = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(FusekiApp.dirConfiguration, baseFilename+".*") ) { + stream.forEach((p)-> paths.add(FusekiApp.dirConfiguration.resolve(p).toString() )); + } + return paths; + } catch (IOException ex) { + throw new InternalErrorException("Failed to read configuration directory "+FusekiApp.dirConfiguration); + } + } + +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ServerMgtConst.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ServerMgtConst.java new file mode 100644 index 00000000000..056b161470c --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/ServerMgtConst.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +/** + * Various constants used in the management API functions and JSON responses in the + * webapp/full server. + */ +public class ServerMgtConst { + public static final String opDatasets = "datasets"; + public static final String opBackup = "backup"; + public static final String opCompact = "compact"; + public static final String opListBackups = "backups-list"; + public static final String opServer = "server"; + + public static final String uptime = "uptime"; + public static final String startDT = "startDateTime"; +// public static final String server = "server"; +// public static final String port = "port"; + public static final String hostname = "hostname"; + public static final String admin = "admin"; + public static final String version = "version"; + public static final String built = "built"; + public static final String services = "services"; +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Template.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Template.java new file mode 100644 index 00000000000..2ef63652b8e --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/Template.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import java.nio.file.Path; + + +public class Template +{ + public static Path getPath(String templateName) { + return FusekiApp.FUSEKI_BASE.resolve(templateName); + } + + public static final String templateDir = "templates"; + + // These are used by the command line start up. + public static final String templateServiceFN = templateDir+"/config-service"; // Dummy used by dataset-less service. + + // TDB1 - for backwards compatibility, the files are called "tdb" + public static final String templateTDB1_FN = templateDir+"/config-tdb"; + public static final String templateTDB1_MemFN = templateDir+"/config-tdb-mem"; + public static final String templateTDB1_DirFN = templateDir+"/config-tdb-dir"; + public static final String templateTDB1_DirReadFN = templateDir+"/config-tdb-dir-read-only"; + + public static final String templateTDB2_FN = templateDir+"/config-tdb2"; + public static final String templateTDB2_MemFN = templateDir+"/config-tdb2-mem"; + public static final String templateTDB2_DirFN = templateDir+"/config-tdb2-dir"; + public static final String templateTDB2_DirReadFN = templateDir+"/config-tdb2-dir-read-only"; + + + public static final String templateTIM_MemFN = templateDir+"/config-mem"; + + // Template may be in a resources area of a jar file so you can't do a directory listing. + public static final String[] templateNames = { + templateTIM_MemFN, + + templateTDB1_FN , + templateTDB1_MemFN , + templateTDB1_DirFN , + //templateTDB1_DirReadFN, + + templateTDB2_FN , + templateTDB2_MemFN , + templateTDB2_DirFN , + //templateTDB2_DirReadFN + }; + + public static final String NAME = "NAME"; + public static final String DATA = "DATA"; + public static final String DIR = "DIR"; +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/TemplateFunctions.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/TemplateFunctions.java new file mode 100644 index 00000000000..009cc655ae6 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mgt/TemplateFunctions.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mgt; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.jena.atlas.io.IO; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.riot.Lang; +import org.apache.jena.util.FileUtils; + +public class TemplateFunctions +{ + /** Read in a template from a file, substitute for {NAME} and return the string. */ + public static String templateFile(String templateName, Map params, Lang lang) { + String templateFilename = Template.getPath(templateName).toString(); + String template; + try { template = FileUtils.readWholeFileAsUTF8(templateFilename); } + catch (IOException ex) { + Fuseki.serverLog.error("File not found: "+templateFilename); + IO.exception(ex); return null; + } + return templateString(template, params, lang); + } + + /** Read a template file, substitute for {NAME} and return the model. */ + public static String templateResource(String resourceName, Map params, Lang lang) { + String template; + try { + InputStream in = TemplateFunctions.class.getClassLoader().getResourceAsStream(resourceName); + if ( in == null ) + Fuseki.serverLog.error("Resource not found: "+resourceName); + template = FileUtils.readWholeFileAsUTF8(in); + } + catch (IOException ex) { + Fuseki.serverLog.error("Error reading resource: "+resourceName); + IO.exception(ex); return null; + } + return templateString(template, params, lang); + } + + /** Create a template from a String */ + public static String templateString(String template, Map params, Lang lang) { + for ( Entry e : params.entrySet() ) { + // Literal string replacement. + // If using .replaceAll, need to use Match.quoteReplacement on the value. + String x = e.getValue(); + String k = "{"+e.getKey()+"}"; + + if ( lang != null ) { + if ( Lang.TTL.equals(lang) || + Lang.TRIG.equals(lang) || + Lang.NT.equals(lang) || + Lang.NQ.equals(lang) || + Lang.JSONLD.equals(lang) || + Lang.RDFJSON.equals(lang) + ) { + // Make safe for a RDF language ""-string - especially MS Windows \ path separators. + x = x.replace("\\", "\\\\"); + x = x.replace("\"", "\\\""); + } + } + template = template.replace(k, x); + } + return template; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/access/FMod_GraphAccessCtl.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/access/FMod_GraphAccessCtl.java new file mode 100644 index 00000000000..fc6e74236a3 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/access/FMod_GraphAccessCtl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.access; + +import org.apache.jena.fuseki.access.DataAccessCtl; +import org.apache.jena.fuseki.access.VocabSecurity; +import org.apache.jena.fuseki.main.FusekiLib; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.server.DataAccessPoint; +import org.apache.jena.rdf.model.Model; + +public class FMod_GraphAccessCtl implements FusekiModule { + + public FMod_GraphAccessCtl() { + VocabSecurity.init(); + } + + @Override + public String name() { + return "GraphAccessCtl"; + } + + @Override + public void configDataAccessPoint(DataAccessPoint dap, Model configModel) { + if ( DataAccessCtl.isAccessControlled(dap.getDataService().getDataset()) ) { + dap.getDataService().forEachEndpoint(ep-> + FusekiLib.modifyForAccessCtl(ep, DataAccessCtl.requestUserServlet)); + } + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ActionServerStatus.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ActionServerStatus.java new file mode 100644 index 00000000000..d4589ede33b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ActionServerStatus.java @@ -0,0 +1,99 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import static org.apache.jena.riot.WebContent.charsetUTF8; +import static org.apache.jena.riot.WebContent.contentTypeJSON; + +import java.io.IOException; +import java.io.OutputStream; + +import org.apache.jena.atlas.json.JSON; +import org.apache.jena.atlas.json.JsonBuilder; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.ctl.JsonDescription; +import org.apache.jena.fuseki.mgt.ServerMgtConst; +import org.apache.jena.fuseki.server.DataAccessPointRegistry; +import org.apache.jena.fuseki.server.ServerConst; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; + +/** Description of datasets for a server */ +public class ActionServerStatus extends ActionCtl +{ + public ActionServerStatus() { super(); } + + @Override + public void validate(HttpAction action) {} + + @Override + public void execGet(HttpAction action) { + executeLifecycle(action); + } + + @Override + public void execPost(HttpAction action) { + executeLifecycle(action); + } + + @Override + public void execute(HttpAction action) { + try { + description(action); + ServletOps.success(action); + } catch (IOException e) { + ServletOps.errorOccurred(e); + } + } + + private void description(HttpAction action) throws IOException { + OutputStream out = action.getResponseOutputStream(); + action.setResponseContentType(contentTypeJSON); + action.setResponseCharacterEncoding(charsetUTF8); + + JsonBuilder builder = new JsonBuilder(); + builder.startObject(); + describeServer(builder, action.getRequestLocalPort()); + describeDatasets(builder, action.getDataAccessPointRegistry()); + builder.finishObject(); + + JsonValue v = builder.build(); + JSON.write(out, v); + out.write('\n'); + out.flush(); + } + + private void describeServer(JsonBuilder builder, int requestPort) { + String versionStr = Fuseki.VERSION; + builder + .pair(ServerMgtConst.version, versionStr) + .pair(ServerMgtConst.startDT, Fuseki.serverStartedAt()) + .pair(ServerMgtConst.uptime, Fuseki.serverUptimeSeconds()) + ; + } + + private void describeDatasets(JsonBuilder builder, DataAccessPointRegistry registry) { + builder.key(ServerConst.datasets); + JsonDescription.arrayDatasets(builder, registry); + } + +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ArgModuleAdmin.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ArgModuleAdmin.java new file mode 100644 index 00000000000..6cb3ed0b4e5 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/ArgModuleAdmin.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.apache.jena.cmd.ArgDecl; +import org.apache.jena.cmd.ArgModuleGeneral; +import org.apache.jena.cmd.CmdArgModule; +import org.apache.jena.cmd.CmdGeneral; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.fuseki.mgt.FusekiApp; + +public class ArgModuleAdmin implements ArgModuleGeneral { + // Add a static of "extra command" + + private ArgDecl argAdmin = new ArgDecl(true, "admin"); + private ArgDecl argAdminArea = new ArgDecl(true, "adminArea", "adminBase"); + + public ArgModuleAdmin() { } + + @Override + public void processArgs(CmdArgModule cmdLine) { + System.out.println("ArgModuleAdmin"); + String admin = cmdLine.getValue(argAdmin); + if ( admin == null ) { + return; + } + + if ( admin.equals("localhost") ) {} + else { + String pwFile = admin; + } + + String dirStr = cmdLine.getValue(argAdminArea); + Path directory = Path.of(dirStr); + + if ( ! Files.isDirectory(directory) ) + throw new FusekiConfigException("Not a directory: "+dirStr); + + if ( ! Files.isWritable(directory) ) + throw new FusekiConfigException("Not writable: "+dirStr); + + FusekiApp.FUSEKI_BASE = directory; + } + + @Override + public void registerWith(CmdGeneral cmdLine) { + cmdLine.add(argAdmin, "--admin=[UserPasswordFile|localhost]", "Enable the admin module"); + cmdLine.add(argAdminArea, "--run=DIR", "Admin state directory"); + } +} \ No newline at end of file diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/FMod_Admin.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/FMod_Admin.java new file mode 100644 index 00000000000..09b0fcffc6a --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/FMod_Admin.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; + +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.cmd.ArgDecl; +import org.apache.jena.cmd.ArgModuleGeneral; +import org.apache.jena.cmd.CmdArgModule; +import org.apache.jena.cmd.CmdGeneral; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.fuseki.build.FusekiConfig; +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.cmds.ServerArgs; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.mgt.ActionBackup; +import org.apache.jena.fuseki.mgt.ActionBackupList; +import org.apache.jena.fuseki.mgt.ActionDatasets; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.fuseki.server.DataAccessPoint; +import org.apache.jena.rdf.model.Model; +import org.slf4j.Logger; + +public class FMod_Admin implements FusekiModule { + + private static FusekiModule singleton = new FMod_Admin(); + // ---- + +// @Override +// public void start() {} +// +// @Override +// public int level() { +// return FusekiApp.levelFModAdmin; +// } + + @Override + public String name() { + return "FMod Admin"; + } + + public static FMod_Admin create() { + return new FMod_Admin(); + } + + public FMod_Admin() {} + + private static Logger LOG = Fuseki.configLog; + + private ArgDecl argAdmin = new ArgDecl(true, "admin"); + private ArgDecl argAdminArea = new ArgDecl(true, "adminArea", "adminBase"); + + @Override + public void serverArgsModify(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + + fusekiCmd.getUsage().startCategory("Admin"); + + ArgModuleGeneral argModule = new ArgModuleGeneral() { + @Override + public void registerWith(CmdGeneral cmdLine) { + cmdLine.add(argAdmin, "--admin", "Enable server admin with user:password"); + cmdLine.add(argAdminArea,"--adminRun", "Directory for server configuration"); + } + @Override + public void processArgs(CmdArgModule cmdLine) {} + }; + argModule.registerWith(fusekiCmd); + } + + @Override + public void serverArgsPrepare(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + String admin = fusekiCmd.getValue(argAdmin); + if ( admin == null ) { + return; + } + + Path directory = null; + String dirStr = fusekiCmd.getValue(argAdminArea); + if ( dirStr != null ) + directory = Path.of(dirStr); + + if ( admin.equals("localhost") ) {} + else { + String pwFile = admin; + } + + if ( directory != null ) { + if ( ! Files.isDirectory(directory) ) + throw new FusekiConfigException("Not a directory: "+dirStr); + + if ( ! Files.isWritable(directory) ) + throw new FusekiConfigException("Not writable: "+dirStr); + } + FusekiApp.FUSEKI_BASE = directory; + } + +// @Override +// public void serverArgsBuilder(FusekiServer.Builder serverBuilder, Model configModel) { +// } + + // ---- + + @Override + public void prepare(FusekiServer.Builder builder, Set datasetNames, Model configModel) { + // Unpack + Path path = FusekiApp.setup(); + + FmtLog.info(LOG, "Fuseki Admin: %s", path); + + // Shiro. + Path shiroIni = path.resolve(FusekiApp.DFT_SHIRO_INI); + if ( Files.exists(shiroIni) ) { + System.setProperty(FusekiApp.envFusekiShiro, shiroIni.toString()); + } else { + FmtLog.info(LOG, "No shiro.ini: dir=%s", path); + } + + String configDir = FusekiApp.dirConfiguration.toString(); + List directoryDatabases = FusekiConfig.readConfigurationDirectory(configDir); + + if ( directoryDatabases.isEmpty() ) + FmtLog.info(LOG, "No databases: dir=%s", configDir); + else { + directoryDatabases.forEach(dap -> FmtLog.info(Fuseki.configLog, "Database: %s", dap.getName())); + } + + directoryDatabases.forEach(db -> { + String dbName = db.getName(); + if ( datasetNames.contains(dbName) ) { + FmtLog.warn(LOG, "Database '%s' already added to the Fuseki server builder", dbName); + // ?? builder.remove(dbName); + } + builder.add(dbName, db.getDataService()); + // ** builder.add(DataAccessPoint); + }); + + // Modify the server to include the admin operations. + // Security is performed by FMod_Shiro. + ActionCtl actionBackup = new ActionBackup(); + builder + .addServlet("/$/datasets/*", new ActionDatasets()) + .addServlet("/$/server", new ActionServerStatus()) + .addServlet("/$/backup/*", actionBackup) + .addServlet("/$/backups/*", actionBackup) + .addServlet("/$/backups-list", new ActionBackupList()) + + // Enables the task subsystem and is also called by enableCompact + .enableTasks(true) + + // Can also be enabled by FMod_UI + .enableStats(true) + .enablePing(true) + .enableCompact(true) + ; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/LocalhostOnly.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/LocalhostOnly.java new file mode 100644 index 00000000000..9429b606508 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/admin/LocalhostOnly.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; + +import jakarta.servlet.*; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.web.HttpSC; +import org.slf4j.Logger; + +/** + * Responds with HTTP 403 on any denied request. + */ +public class LocalhostOnly implements Filter { + + private static String LOCALHOST_IpV6_a = "[0:0:0:0:0:0:0:1]"; + private static String LOCALHOST_IpV6_b = "0:0:0:0:0:0:0:1"; + // This is what appears in the Chrome developer tools client-side. + // "[0:0:0:0:0:0:0:1]" by the time it arrives here, it is not clear which + // software component is responsible for that. + // To be safe we add "[::1]". + private static String LOCALHOST_IpV6_c = "[::1]"; + private static String LOCALHOST_IpV4 = "127.0.0.1"; // Strictly, 127.*.*.* + + private static final Collection localhosts = new HashSet<>( + Arrays.asList(LOCALHOST_IpV4, LOCALHOST_IpV6_a, LOCALHOST_IpV6_b, LOCALHOST_IpV6_c)); + + private static Logger log = Fuseki.serverLog; + private static final String message = "Access denied : only localhost access allowed"; + + public LocalhostOnly() { } + + // "permit" and "deny" lists + private List secured = Arrays.asList("/$/backup", "/$/compact", "/$/datasets"); + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { + try { + HttpServletRequest req = (HttpServletRequest)request; + HttpServletResponse resp = (HttpServletResponse)response; + /** + * Check request from "localhost, else 403. + */ + boolean accept = checkRequest(req, resp); + if ( ! accept ) { + // Log + resp.sendError(HttpSC.FORBIDDEN_403); + return; + } + } catch (Throwable ex) { + log.info("Filter: unexpected exception: "+ex.getMessage(),ex); + } + // Continue. + chain.doFilter(request, response); + } + + public boolean checkRequest(HttpServletRequest req, HttpServletResponse resp) { + String uri = req.getRequestURI(); + for ( String s : secured ) { + if ( uri.startsWith(s) ) { + if ( ! checkLocalhost(req) ) + return false; + } + } + return true; + } + + public static boolean checkLocalhost(HttpServletRequest request) { + return localhosts.contains(request.getRemoteAddr()) ; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/blank/FMod_BLANK.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/blank/FMod_BLANK.java new file mode 100644 index 00000000000..4b56d2d7b17 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/blank/FMod_BLANK.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.blank; + +import org.apache.jena.fuseki.main.sys.FusekiAutoModule; + +/** + * Template. + * + * src/main/resources/META-INF/services/org.apache.jena.fuseki.main.sys.FusekiAutoModule + */ +public class FMod_BLANK implements FusekiAutoModule { + @Override + public String name() { return "BLANK"; } + +// @Override public void start() { } +// @Override public void prepare(FusekiServer.Builder serverBuilder, Set datasetNames, Model configModel) { } +// @Override public void configured(DataAccessPointRegistry dapRegistry, Model configModel) { +// dapRegistry.accessPoints().forEach(accessPoint->configDataAccessPoint(accessPoint, configModel)); +// } +// @Override public void configDataAccessPoint(DataAccessPoint dap, Model configModel) {} +// @Override public void server(FusekiServer server) { } +// @Override public void serverBeforeStarting(FusekiServer server) { } +// @Override public void serverAfterStarting(FusekiServer server) { } +// @Override public void serverStopped(FusekiServer server) { } +// @Override public void stop() {} +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/package-info.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/package-info.java new file mode 100644 index 00000000000..7345d58a493 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/package-info.java @@ -0,0 +1,2 @@ +package org.apache.jena.fuseki.mod; + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/ActionMetrics.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/ActionMetrics.java new file mode 100644 index 00000000000..92164563a6a --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/ActionMetrics.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.fuseki.mod.prometheus; + +import org.apache.jena.fuseki.ctl.ActionCtl; +import org.apache.jena.fuseki.metrics.MetricsProviderRegistry; +import org.apache.jena.fuseki.servlets.ActionLib; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; + +public class ActionMetrics extends ActionCtl { + + public ActionMetrics() { super(); } + + @Override + public void execGet(HttpAction action) { + super.executeLifecycle(action); + } + + @Override + public void execOptions(HttpAction action) { + ActionLib.doOptionsGet(action); + ServletOps.success(action); + } + + @Override + public void validate(HttpAction action) {} + + @Override + public void execute(HttpAction action) { + MetricsProviderRegistry.get().scrape( action ); + ServletOps.success(action); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/FMod_Prometheus.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/FMod_Prometheus.java new file mode 100644 index 00000000000..0d51e0bd4fd --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/FMod_Prometheus.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.prometheus; + +import java.util.Set; + +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.metrics.MetricsProviderRegistry; +import org.apache.jena.rdf.model.Model; + +/** + * Prometheus Metrics. + * + * PrometheusMetricsProvider + */ +public class FMod_Prometheus implements FusekiModule { + + private static FusekiModule singleton = new FMod_Prometheus(); + public static FusekiModule get() { + return singleton; + } + + public FMod_Prometheus() {} + +// @Override +// public int level() { +// return 5000; +// } +// +// @Override public void start() { +// Fuseki.configLog.info("FMod Prometheus Metrics"); +// MetricsProviderRegistry.set(new PrometheusMetricsProvider()); +// } + + @Override + public String name() { return "FMod Prometheus Metrics"; } + + @Override public void prepare(FusekiServer.Builder serverBuilder, Set datasetNames, Model configModel) { + //MetricsProviderRegistry.set(new PrometheusMetricsProvider()); + serverBuilder.addServlet("/$/metrics", new ActionMetrics()); + } + + @Override public void server(FusekiServer server) { + MetricsProviderRegistry.dataAccessPointMetrics(server.getDataAccessPointRegistry()); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/PrometheusMetricsProvider.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/PrometheusMetricsProvider.java new file mode 100644 index 00000000000..74e13fadc1f --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/prometheus/PrometheusMetricsProvider.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jena.fuseki.mod.prometheus; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.prometheusmetrics.PrometheusConfig; +import io.micrometer.prometheusmetrics.PrometheusMeterRegistry; +import jakarta.servlet.ServletOutputStream; +import org.apache.jena.fuseki.metrics.FusekiMetrics; +import org.apache.jena.fuseki.metrics.MetricsProvider; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.riot.WebContent; + +/** + */ +public class PrometheusMetricsProvider implements MetricsProvider { + + private PrometheusMeterRegistry meterRegistry; + + public PrometheusMetricsProvider() { + meterRegistry = new PrometheusMeterRegistry( PrometheusConfig.DEFAULT ); + meterRegistry.config().commonTags( "application", "fuseki" ); + FusekiMetrics.registerMetrics(meterRegistry); + } + + @Override + public MeterRegistry getMeterRegistry() { + return meterRegistry; + } + + @Override + public void scrape(HttpAction action) { + try (ServletOutputStream out = action.getResponseOutputStream()) { + ServletOps.success(action); + action.setResponseContentType( WebContent.contentTypeTextPlain ); + action.setResponseCharacterEncoding( WebContent.charsetUTF8 ); + + out.write( meterRegistry.scrape().getBytes() ); + } catch (Throwable t) { + ServletOps.errorOccurred( t ); + } + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FMod_Shiro.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FMod_Shiro.java new file mode 100644 index 00000000000..e70a44d58d4 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FMod_Shiro.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.shiro; + +import java.nio.file.Path; +import java.util.List; +import java.util.Set; + +import jakarta.servlet.Filter; +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.atlas.lib.IRILib; +import org.apache.jena.atlas.lib.Lib; +import org.apache.jena.cmd.ArgDecl; +import org.apache.jena.cmd.CmdException; +import org.apache.jena.cmd.CmdGeneral; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.cmds.ServerArgs; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.rdf.model.Model; +import org.apache.shiro.web.servlet.ShiroFilter; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.SessionHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Fuseki Module for Apache Shiro. + *

+ * TODO + * Configuration + */ +public class FMod_Shiro implements FusekiModule { + +// @Override +// public void start() { +// Fuseki.serverLog.info("FMod Shiro"); +// } + // +// @Override +// public int level() { +// return FusekiApp.levelFModShiro; +// } + + @Override + public String name() { + return "FMod Shiro"; + } + + public static FMod_Shiro create() { + return new FMod_Shiro(); + } + + // Assumes the whole system is "Shiro". + // No setup? + + public static final Logger shiroConfigLog = LoggerFactory.getLogger(Fuseki.PATH + ".Shiro"); + + private static List defaultIniFileLocations = List.of("file:shiro.ini", "file:/etc/fuseki/shiro.ini"); + private static List iniFileLocations = null; + + private static ArgDecl argShiroIni = new ArgDecl(true, "shiro", "shiro-ini"); + + // XXX Should be a per build variable. + private String shiroFile = null; + + public FMod_Shiro() { + this(null); + } + + public FMod_Shiro(String shiroFile) { + this.shiroFile = shiroFile; + } + + // ---- If used from the command line + @Override + public void serverArgsModify(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + fusekiCmd.add(argShiroIni); + } + + @Override + public void serverArgsPrepare(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + if ( fusekiCmd.contains(argShiroIni) ) { + shiroFile = fusekiCmd.getValue(argShiroIni); + Path path = Path.of(shiroFile); + IOX.checkReadableFile(path, CmdException::new); + } + } + + // The filter is added in prepare(). + // This allows other Fuseki modules, such as FMod_Admin, to setup shiro.ini. + // FMod_Admin unpacks a default one to $FUSEKI_BASE/shiro.ini (usually "run/shiro.ini") + +// @Override +// public void serverArgsBuilder(FusekiServer.Builder serverBuilder, Model configModel) { +// //Add filter. +// } + + /** + * Determine the Shiro configuration file. + * This applies whether command line arguments used for programmatic setup. + */ + @Override + public void prepare(FusekiServer.Builder serverBuilder, Set datasetNames, Model configModel) { + if ( shiroFile == null ) { + // Environment variable: FUSEKI_SHIRO + shiroFile = Lib.getenv(FusekiApp.envFusekiShiro); + } + + if ( shiroFile == null ) { + return; + } + + if ( shiroFile != null ) { + IOX.checkReadableFile(shiroFile, FusekiConfigException::new); + Filter filter = new FusekiShiroFilter(shiroFile); + // This is a "before" filter. + serverBuilder.addFilter("/*", filter); + } + + // Clear. + shiroFile = null; + } + + /** + * FusekiShiroFilter, includes Shiro initialization. Fuseki is a + * not a webapp so it needs to trigger off servlet initialization. + */ + private static class FusekiShiroFilter extends ShiroFilter { + + private final String shiroInitializationFile; + + FusekiShiroFilter(String filename) { + shiroInitializationFile = IRILib.filenameToIRI(filename); + } + + @Override + public void init() throws Exception { + // Intercept Shiro initialization. + List locations = List.of(); + if ( shiroInitializationFile != null ) { + locations = List.of(shiroInitializationFile); + } + FusekiShiroLib.shiroEnvironment(getServletContext(), locations); + super.init(); + } + } + + @Override + public void serverBeforeStarting(FusekiServer server) { + // Shiro requires a session handler. + // This needs the Jetty server to have been created. + org.eclipse.jetty.server.Server jettyServer = server.getJettyServer(); + try { + ServletContextHandler servletContextHandler = (ServletContextHandler)(jettyServer.getHandler()); + if ( servletContextHandler.getSessionHandler() == null ) { + SessionHandler sessionsHandler = new SessionHandler(); + servletContextHandler.setHandler(sessionsHandler); + } + } catch (RuntimeException ex) { + shiroConfigLog.error("Failed to set a session manager - server aborted"); + throw ex; + } + } + + @Override + public void serverAfterStarting(FusekiServer server) {} + +// @Override public void serverStopped(FusekiServer server) { } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FusekiShiroLib.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FusekiShiroLib.java new file mode 100644 index 00000000000..c963771efcb --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/FusekiShiroLib.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.shiro; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import jakarta.servlet.ServletContext; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.rfc3986.URIScheme; +import org.apache.shiro.lang.io.ResourceUtils; +import org.apache.shiro.web.env.EnvironmentLoaderListener; + +/*package*/ class FusekiShiroLib { + static void shiroEnvironment(ServletContext servletContext, List possibleShiroIniFiles) { + // Shiro environment initialization, done here because we don't have webapp listeners. + EnvironmentLoaderListener shiroListener = new ShiroEnvironmentLoaderListener(possibleShiroIniFiles); + try { + shiroListener.initEnvironment(servletContext); + } catch (org.apache.shiro.config.ConfigurationException ex) { + ShiroEnvironmentLoaderListener.shiroConfigLog.error("Failed to initialize Shiro: "+ex.getMessage()); + throw new FusekiConfigException(ex.getMessage()); + } + } + + private static String fileSchemePrefix = URIScheme.FILE.getPrefix(); + + /** Look for a Shiro ini file, returning the first found, or return null */ + static String huntForShiroIni(List locations) { + for ( String loc : locations ) { + // If file:, look for that file. + if ( loc.startsWith(fileSchemePrefix) ) { + Path p = Path.of(loc.substring(fileSchemePrefix.length())); + if ( Files.exists(p) ) + return loc; + // Ignore. + continue; + } + // No scheme. May be a classpath resource. + if ( ResourceUtils.resourceExists(loc) ) + return loc; + } + return null; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/ShiroEnvironmentLoaderListener.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/ShiroEnvironmentLoaderListener.java new file mode 100644 index 00000000000..c99f47aee71 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/shiro/ShiroEnvironmentLoaderListener.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.shiro; + +import static java.lang.String.format; + +import java.util.Arrays; +import java.util.List; + +import org.apache.shiro.web.env.EnvironmentLoaderListener; +import org.apache.shiro.web.env.ResourceBasedWebEnvironment; +import org.apache.shiro.web.env.WebEnvironment; +import org.slf4j.Logger; + +/** + * A Shiro {@link EnvironmentLoaderListener} that supports multiple possible + * locations for a {@code shiro.ini} file. It will return the first found in a list + * of possible file names. + */ +class ShiroEnvironmentLoaderListener extends EnvironmentLoaderListener{ + + public static final Logger shiroConfigLog = FMod_Shiro.shiroConfigLog; + + private List locations; + + /*package*/ ShiroEnvironmentLoaderListener(List locations) { + this.locations = locations; + } + + /** + * When given multiple locations for the shiro.ini file, and + * if a {@link ResourceBasedWebEnvironment}, check the list of configuration + * locations, testing whether the name identified an existing resource. + * For the first resource name found to exist, reset the {@link ResourceBasedWebEnvironment} + * to name that resource alone so the normal Shiro initialization executes. + */ + @Override + protected void customizeEnvironment(WebEnvironment environment) { + if ( locations == null ) + return; + + // Look for shiro.ini + if ( environment instanceof ResourceBasedWebEnvironment ) { + ResourceBasedWebEnvironment env = (ResourceBasedWebEnvironment)environment; + String[] configLocations = env.getConfigLocations(); + if ( configLocations != null && configLocations.length > 0 ) { + // Set some other way. + shiroConfigLog.info(format("Shiro file resource %s", Arrays.asList(configLocations))); + return; + } + String loc = FusekiShiroLib.huntForShiroIni(locations); + if ( loc == null ) { + shiroConfigLog.info(format("No Shiro file found (tried: %s)", locations)); + return; + } + shiroConfigLog.info("Shiro INI: "+loc); + String[] configLocationsHere = new String[] {loc}; + env.setConfigLocations(configLocationsHere); + } + } +} \ No newline at end of file diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStats.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStats.java new file mode 100644 index 00000000000..51953d15682 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStats.java @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.ui; + +import static java.lang.String.format; + +import java.util.List; + +import org.apache.commons.lang3.StringUtils; +import org.apache.jena.atlas.json.JsonBuilder; +import org.apache.jena.atlas.json.JsonObject; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.fuseki.ctl.ActionContainerItem; +import org.apache.jena.fuseki.server.*; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; + +/** + * JSON output for stats, than JSON {@link ActionStats}. + * Separate because content negotiation for known JSON may + * be a simple "Accept: {@literal *}/*}". + *

+ * 

+ *   /$/serviceurl -- all datatsets
+ *   /$/serviceurl/name -- one dataset
+ *
+ * + */ +public class ActionStats extends ActionContainerItem +{ + // For endpoint with "" as name. + private static String emptyNameKeyPrefix = "_"; + + public ActionStats() { super(); } + + @Override + public void validate(HttpAction action) {} + + @Override + protected JsonValue execPostContainer(HttpAction action) { + return execCommonContainer(action); + } + + @Override + protected JsonValue execPostItem(HttpAction action) { + return execCommonItem(action); + } + + @Override + protected JsonValue execGetContainer(HttpAction action) { + return execCommonContainer(action); + } + + @Override + protected JsonValue execGetItem(HttpAction action) { + return execCommonItem(action); + } + + // This does not consult the system database for dormant etc. + protected JsonValue execCommonContainer(HttpAction action) { + if ( action.verbose ) + action.log.info(format("[%d] GET stats all", action.id)); + return generateStats(action.getDataAccessPointRegistry()); + } + + public static JsonObject generateStats(DataAccessPointRegistry registry) { + JsonBuilder builder = new JsonBuilder(); + builder.startObject("top"); + builder.key(ServerConst.datasets); + builder.startObject("datasets"); + registry.forEach((name, access)->statsDataset(builder, access)); + builder.finishObject("datasets"); + builder.finishObject("top"); + return builder.build().getAsObject(); + } + + protected JsonValue execCommonItem(HttpAction action) { + String datasetPath = getItemDatasetName(action); + if ( action.verbose ) + action.log.info(format("[%d] GET stats dataset %s", action.id, datasetPath)); + + JsonBuilder builder = new JsonBuilder(); + DataAccessPoint dap = getItemDataAccessPoint(action, datasetPath); + if ( dap == null ) + ServletOps.errorNotFound(datasetPath); + builder.startObject("TOP"); + + builder.key(ServerConst.datasets); + builder.startObject("datasets"); + statsDataset(builder, datasetPath, action.getDataAccessPointRegistry()); + builder.finishObject("datasets"); + + builder.finishObject("TOP"); + return builder.build(); + } + + public static JsonObject generateStats(DataAccessPoint access) { + JsonBuilder builder = new JsonBuilder(); + statsDataset(builder, access); + return builder.build().getAsObject(); + } + + private void statsDataset(JsonBuilder builder, String name, DataAccessPointRegistry registry) { + DataAccessPoint access = registry.get(name); + statsDataset(builder, access); + } + + private static void statsDataset(JsonBuilder builder, DataAccessPoint access) { + // Object started + builder.key(access.getName()); + DataService dSrv = access.getDataService(); + builder.startObject("counters"); + + builder.key(CounterName.Requests.getName()).value(dSrv.getCounters().value(CounterName.Requests)); + builder.key(CounterName.RequestsGood.getName()).value(dSrv.getCounters().value(CounterName.RequestsGood)); + builder.key(CounterName.RequestsBad.getName()).value(dSrv.getCounters().value(CounterName.RequestsBad)); + + builder.key(ServerConst.endpoints).startObject("endpoints"); + int unique = 0; + for ( Operation operName : dSrv.getOperations() ) { + List endpoints = access.getDataService().getEndpoints(operName); + for ( Endpoint endpoint : endpoints ) { + String k = endpoint.getName(); + if ( StringUtils.isEmpty(k) ) + k = emptyNameKeyPrefix+(++unique); + // Endpoint names are unique for a given service. + builder.key(k); + builder.startObject(); + + operationCounters(builder, endpoint); + builder.key(ServerConst.operation).value(operName.getJsonName()); + builder.key(ServerConst.description).value(operName.getDescription()); + + builder.finishObject(); + } + } + builder.finishObject("endpoints"); + builder.finishObject("counters"); + } + + private static void operationCounters(JsonBuilder builder, Endpoint operation) { + for (CounterName cn : operation.getCounters().counters()) { + Counter c = operation.getCounters().get(cn); + builder.key(cn.getName()).value(c.value()); + } + } +} + + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStatsTxt.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStatsTxt.java new file mode 100644 index 00000000000..7b2bbe7ab67 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/ActionStatsTxt.java @@ -0,0 +1,189 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.ui; + +import static java.lang.String.format; + +import java.io.IOException; +import java.util.Iterator; + +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.fuseki.ctl.ActionContainerItem; +import org.apache.jena.fuseki.server.*; +import org.apache.jena.fuseki.servlets.HttpAction; +import org.apache.jena.fuseki.servlets.ServletOps; +import org.apache.jena.riot.WebContent; + +/** + * Text output for stats, than JSON {@link ActionStats}. + * Separate because content negotiation for known JSON may + * be a simple "Accept: {@literal *}/*}". + *

+ * 

+ *   /$/serviceurl -- all datatsets
+ *   /$/serviceurl/name -- one dataset
+ *
+ * + */ +public class ActionStatsTxt extends ActionContainerItem { + + public ActionStatsTxt() { super(); } + + @Override + public void validate(HttpAction action) {} + + // Stats as plain text. + // Not simple conneg because JSON can be wanted but the HRRP is "Accept */*" + // Probably replace with a "simple" view. + + // All datasets + @Override + protected JsonValue execGetContainer(HttpAction action) { + execContainer(action); + return null; + } + + // One dataset + @Override + protected JsonValue execGetItem(HttpAction action) { + execItem(action); + return null; + } + + // POST is GET (but know to be fresh) + @Override + protected JsonValue execPostContainer(HttpAction action) { + execContainer(action); + return null; + } + + @Override + protected JsonValue execPostItem(HttpAction action) { + execItem(action); + return null; + } + + private void execContainer(HttpAction action) { + if ( action.verbose ) + action.log.info(format("[%d] GET stats text", action.id)); + try { + statsTxt(action.getResponse(), action.getDataAccessPointRegistry()); + } catch (IOException ex) { + action.log.warn(format("[%d] GET stats text: IO error: %s", action.id, ex.getMessage())); + } + } + + private void execItem(HttpAction action) { + String name = getItemDatasetName(action); + if ( name == null ) + ServletOps.errorBadRequest("No dataset name"); + DataAccessPoint desc = action.getDataAccessPointRegistry().get(name); + if ( desc == null ) + ServletOps.errorBadRequest("No such dataset: "+name); + try { + HttpServletResponse resp = action.getResponse(); + ServletOutputStream out = resp.getOutputStream(); + resp.setContentType(WebContent.contentTypeTextPlain); + resp.setCharacterEncoding(WebContent.charsetUTF8); + statsTxt(out, desc); + } catch (IOException ex) { + action.log.warn(format("[%d] GET stats text: IO error: %s", action.id, ex.getMessage())); + } + } + + private void statsTxt(HttpServletResponse resp, DataAccessPointRegistry registry) throws IOException { + ServletOutputStream out = resp.getOutputStream(); + resp.setContentType(WebContent.contentTypeTextPlain); + resp.setCharacterEncoding(WebContent.charsetUTF8); + + Iterator iter = registry.keys().iterator(); + while (iter.hasNext()) { + String ds = iter.next(); + DataAccessPoint desc = registry.get(ds); + statsTxt(out, desc); + if ( iter.hasNext() ) + out.println(); + } + out.flush(); + } + + private void statsTxt(ServletOutputStream out, DataAccessPoint desc) throws IOException { + DataService dSrv = desc.getDataService(); + out.println("Dataset: " + desc.getName()); + out.println(" Requests = " + dSrv.getCounters().value(CounterName.Requests)); + out.println(" Good = " + dSrv.getCounters().value(CounterName.RequestsGood)); + out.println(" Bad = " + dSrv.getCounters().value(CounterName.RequestsBad)); + + if ( desc.getDataService().hasOperation(Operation.Query)) { + out.println(" SPARQL Query:"); + out.println(" Request = " + counter(dSrv, Operation.Query, CounterName.Requests)); + out.println(" Good = " + counter(dSrv, Operation.Query, CounterName.RequestsGood)); + out.println(" Bad requests = " + counter(dSrv, Operation.Query, CounterName.RequestsBad)); + out.println(" Timeouts = " + counter(dSrv, Operation.Query, CounterName.QueryTimeouts)); + out.println(" Bad exec = " + counter(dSrv, Operation.Query, CounterName.QueryExecErrors)); + //out.println(" IO Errors = " + counter(dSrv, Operation.Query, CounterName.QueryIOErrors)); + } + + if ( desc.getDataService().hasOperation(Operation.Update)) { + out.println(" SPARQL Update:"); + out.println(" Request = " + counter(dSrv, Operation.Update, CounterName.Requests)); + out.println(" Good = " + counter(dSrv, Operation.Update, CounterName.RequestsGood)); + out.println(" Bad requests = " + counter(dSrv, Operation.Update, CounterName.RequestsBad)); + out.println(" Bad exec = " + counter(dSrv, Operation.Update, CounterName.UpdateExecErrors)); + } + + if ( desc.getDataService().hasOperation(Operation.Upload)) { + out.println(" Upload:"); + out.println(" Requests = " + counter(dSrv, Operation.Upload, CounterName.Requests)); + out.println(" Good = " + counter(dSrv, Operation.Upload, CounterName.RequestsGood)); + out.println(" Bad = " + counter(dSrv, Operation.Upload, CounterName.RequestsBad)); + } + + if ( desc.getDataService().hasOperation(Operation.GSP_R) || desc.getDataService().hasOperation(Operation.GSP_RW) ) { + out.println(" SPARQL Graph Store Protocol:"); + out.println(" GETs = " + gspValue(dSrv, CounterName.HTTPget) + " (good=" + gspValue(dSrv, CounterName.HTTPgetGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPgetBad) + ")"); + if ( desc.getDataService().hasOperation(Operation.GSP_RW) ) { + out.println(" PUTs = " + gspValue(dSrv, CounterName.HTTPput) + " (good=" + gspValue(dSrv, CounterName.HTTPputGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPputBad) + ")"); + out.println(" POSTs = " + gspValue(dSrv, CounterName.HTTPpost) + " (good=" + gspValue(dSrv, CounterName.HTTPpostGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPpostBad) + ")"); + out.println(" PATCHs = " + gspValue(dSrv, CounterName.HTTPpatch) + " (good=" + gspValue(dSrv, CounterName.HTTPpatchGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPpatchBad) + ")"); + out.println(" DELETEs = " + gspValue(dSrv, CounterName.HTTPdelete) + " (good=" + gspValue(dSrv, CounterName.HTTPdeleteGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPdeleteBad) + ")"); + } + out.println(" HEADs = " + gspValue(dSrv, CounterName.HTTPhead) + " (good=" + gspValue(dSrv, CounterName.HTTPheadGood) + + "/bad=" + gspValue(dSrv, CounterName.HTTPheadBad) + ")"); + } + } + + private long counter(DataService dSrv, Operation operation, CounterName cName) { + return 0; + } + + private long gspValue(DataService dSrv, CounterName cn) { + return counter(dSrv, Operation.GSP_RW, cn) + + counter(dSrv, Operation.GSP_R, cn); + } +} + + diff --git a/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/FMod_UI.java b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/FMod_UI.java new file mode 100644 index 00000000000..a46ec343aa5 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/java/org/apache/jena/fuseki/mod/ui/FMod_UI.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.ui; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.atlas.logging.FmtLog; +import org.apache.jena.cmd.ArgDecl; +import org.apache.jena.cmd.CmdException; +import org.apache.jena.cmd.CmdGeneral; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.FusekiConfigException; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.cmds.ServerArgs; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.fuseki.validation.DataValidator; +import org.apache.jena.fuseki.validation.IRIValidator; +import org.apache.jena.fuseki.validation.QueryValidator; +import org.apache.jena.fuseki.validation.UpdateValidator; +import org.apache.jena.rdf.model.Model; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; +import org.slf4j.Logger; + +public class FMod_UI implements FusekiModule { + + // Only one module needed - it is stateless. + private static FusekiModule singleton = new FMod_UI(); + public static FusekiModule get() { + return singleton; + } + + public FMod_UI() {} + + private static Logger LOG = Fuseki.configLog; + +// // After FMod_admin +// @Override +// public int level() { +// return FusekiApp.levelFModUI; +// } + + private static ArgDecl argUIFiles = new ArgDecl(true, "ui"); + private String uiAppLocation = null; + /** Java resource name used to find the UI files. */ + private static String resourceNameUI = "webapp"; + /** Directory name of the root of UI files with {@code FUSEKI_BASE} */ + private static String directoryNameUI = "webapp"; + + @Override + public String name() { + return "FMod UI"; + } + + // ---- If used from the command line + @Override + public void serverArgsModify(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + fusekiCmd.add(argUIFiles); + } + + @Override + public void serverArgsPrepare(CmdGeneral fusekiCmd, ServerArgs serverArgs) { + if ( fusekiCmd.contains(argUIFiles) ) { + uiAppLocation = fusekiCmd.getValue(argUIFiles); + IOX.checkReadableDirectory(uiAppLocation, CmdException::new); + } + } + + @Override + public void prepare(FusekiServer.Builder builder, Set datasetNames, Model configModel) { + if ( builder.staticFileBase() != null ) { + FmtLog.warn(LOG, "Static content location has already been set: %s", builder.staticFileBase()); + return; + } + + if ( uiAppLocation == null ) { + uiAppLocation = findFusekiApp(); + if ( uiAppLocation == null ) { + LOG.warn("No Static content location has been found"); + return; + } + } else { + FmtLog.info(LOG, "UI file area = %s", uiAppLocation); + } + + builder.staticFileBase(uiAppLocation) // Set the UI files area. + .addServlet("/$/validate/query", new QueryValidator()) + .addServlet("/$/validate/update", new UpdateValidator()) + .addServlet("/$/validate/iri", new IRIValidator()) + .addServlet("/$/validate/data", new DataValidator()) + .enableStats(true); + // LOG.info("Fuseki UI loaded"); + } + + /** + * Locate the UI files. + *
    + *
  1. Command line name of a directory
    2. + *
  2. {@code $FUSEKI_BASE/webapp}
    3. + *
  3. Classpath java resource {@code webapp}
    4. + *
      + */ + private String findFusekiApp() { + // 1:: Command line setting. + if ( uiAppLocation != null ) + return uiAppLocation; + + // 2:: $FUSEKI_BASE/webapp + // If the FUSEKI_BASE does not exists, it is created later in FMod_admin.prepare + // and does not include Fuseki app. + String x = fromPath(FusekiApp.FUSEKI_BASE, directoryNameUI); + if ( x != null ) { + LOG.info("Fuseki UI - path resource: "+x); + return x; + } + + // 3:: From a jar. + // Format jar:file:///.../jena-fuseki-ui-VERSION.jar!/webapp/" + String r = fromClasspath(resourceNameUI); + if ( r != null ) { + // Simplify name. + String displayName = loggingName(r); + FmtLog.info(LOG, "UI Base = %s", displayName); + return r; + } + // Bad! + return null; + } + + // Look for "$resourceName" on the classpath. + private static String fromClasspath(String resourceName) { + // Jetty 12.0.15 => warning "Leaked mount" + // Logger : "org.eclipse.jetty.util.resource.ResourceFactory" + //ResourceFactory resourceFactory = ResourceFactory.root(); + + ResourceFactory resourceFactory = ResourceFactory.closeable(); + Resource resource = resourceFactory.newClassLoaderResource(resourceName); + if ( resource != null ) + return resource.getURI().toString(); + return null; + } + + // Look for "$path/$resourceName" + private static String fromPath(Path path, String resourceName) { + if ( path != null ) { + Path path2 = path.resolve(resourceName); + if ( Files.exists(path2) ) { + IOX.checkReadableDirectory(path2, FusekiConfigException::new); + return path2.toAbsolutePath().toString(); + } + } + return null; + } + + private static Pattern regex = Pattern.compile("([^/]*)!"); + + private String loggingName(String r) { + Matcher matcher = regex.matcher(r); + if ( ! matcher.find() ) + return r; + return matcher.group(1); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/config.ttl b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/config.ttl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/shiro.ini b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/shiro.ini new file mode 100644 index 00000000000..72fa8942eb2 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/shiro.ini @@ -0,0 +1,38 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +[main] +# Development +ssl.enabled = false + +plainMatcher=org.apache.shiro.authc.credential.SimpleCredentialsMatcher +localhostFilter=org.apache.jena.fuseki.authz.LocalhostFilter + +[users] +# Implicitly adds "iniRealm = org.apache.shiro.realm.text.IniRealm" +## admin=pw +user1=passwd1 + +[roles] + +[urls] +## Control functions open to anyone +/$/status = anon +/$/server = anon +/$/ping = anon +/$/metrics = anon + +## and the rest are restricted to localhost. +/$/** = localhostFilter + + +## If you want simple, basic authentication user/password +## on the operations, +## 1 - set a better password in [users] above. +## 2 - comment out the "/$/** = localhost" line and use: +##/$/** = authcBasic,user[admin] + +## or to allow any access. +##/$/** = anon + +# Everything else +/**=anon diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-mem b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-mem new file mode 100644 index 00000000000..6f42e594b4c --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-mem @@ -0,0 +1,29 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . + +@prefix rdfs: . +@prefix tdb: . +@prefix ja: . + +## --------------------------------------------------------------- +## Updatable in-memory dataset. + +<#service1> rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/{NAME} + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#dataset> ; + . + +# Transactional, in-memory dataset. Initially empty. +<#dataset> rdf:type ja:DatasetTxnMem . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb new file mode 100644 index 00000000000..9b185884e2b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb @@ -0,0 +1,32 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . +@prefix rdfs: . +@prefix tdb: . +@prefix ja: . + +## --------------------------------------------------------------- +## Updatable TDB dataset with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "query" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + # A separate read-only graph store endpoint: + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + . + +<#tdb_dataset_readwrite> rdf:type tdb:DatasetTDB ; + tdb:location "{FUSEKI_BASE}/databases/{NAME}" ; + ##ja:context [ ja:cxtName "arq:queryTimeout" ; ja:cxtValue "3000" ] ; + ##tdb:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-dir b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-dir new file mode 100644 index 00000000000..f22de072b53 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-dir @@ -0,0 +1,30 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . +@prefix rdfs: . +@prefix tdb: . +@prefix ja: . + +## --------------------------------------------------------------- +## Updatable TDB dataset with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + . + +<#tdb_dataset_readwrite> rdf:type tdb:DatasetTDB ; + tdb:location "{DIR}" ; + ##tdb:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-mem b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-mem new file mode 100644 index 00000000000..6b9c65653bd --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb-mem @@ -0,0 +1,30 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . +@prefix rdfs: . +@prefix tdb: . +@prefix ja: . + +## --------------------------------------------------------------- +## Updatable TDB dataset im-memory with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + . + +<#tdb_dataset_readwrite> rdf:type tdb:DatasetTDB ; + tdb:location "--mem--" ; + ## tdb:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2 b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2 new file mode 100644 index 00000000000..3fa7c166089 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2 @@ -0,0 +1,32 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: +PREFIX rdfs: +PREFIX tdb2: +PREFIX ja: + +## --------------------------------------------------------------- +## Updatable TDB dataset with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB2 {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + + . + +<#tdb_dataset_readwrite> rdf:type tdb2:DatasetTDB2 ; + tdb2:location "{FUSEKI_BASE}/databases/{NAME}" ; + ##ja:context [ ja:cxtName "arq:queryTimeout" ; ja:cxtValue "3000" ] ; + ##tdb2:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-dir b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-dir new file mode 100644 index 00000000000..3312b9348cd --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-dir @@ -0,0 +1,31 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: +PREFIX rdfs: +PREFIX tdb2: +PREFIX ja: + +## --------------------------------------------------------------- +## Updatable TDB2 dataset with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB2 {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + + . + +<#tdb_dataset_readwrite> rdf:type tdb2:DatasetTDB2 ; + tdb2:location "{DIR}" ; + ##tdb2:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-mem b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-mem new file mode 100644 index 00000000000..de362d09d8b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/main/resources/org/apache/jena/fuseki/server/templates/config-tdb2-mem @@ -0,0 +1,30 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: +PREFIX rdfs: +PREFIX tdb2: +PREFIX ja: + +## --------------------------------------------------------------- +## Updatable TDB2 dataset im-memory with all services enabled. + +<#service_tdb_all> rdf:type fuseki:Service ; + rdfs:label "TDB2 {NAME}" ; + fuseki:name "{NAME}" ; + fuseki:serviceQuery "" ; + fuseki:serviceQuery "sparql" ; + fuseki:serviceQuery "query" ; + fuseki:serviceUpdate "" ; + fuseki:serviceUpdate "update" ; + fuseki:serviceUpload "upload" ; + fuseki:serviceReadWriteGraphStore "data" ; + fuseki:serviceReadGraphStore "get" ; + fuseki:dataset <#tdb_dataset_readwrite> ; + . + +<#tdb_dataset_readwrite> rdf:type tdb2:DatasetTDB2 ; + tdb2:location "--mem--" ; + ## tdb2:unionDefaultGraph true ; + . diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/TC_FusekiServer.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/TC_FusekiServer.java new file mode 100644 index 00000000000..fd30ecb9460 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/TC_FusekiServer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki; + +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; + +import org.apache.jena.fuseki.main.TC_FusekiMain; +import org.apache.jena.fuseki.mod.TC_FusekiMods; + +@Suite +@SelectClasses({ + TC_FusekiMain.class, + TC_FusekiMods.class +}) +public class TC_FusekiServer {} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TC_FusekiMods.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TC_FusekiMods.java new file mode 100644 index 00000000000..d2fafda62f6 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TC_FusekiMods.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod; + +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; + +import org.apache.jena.fuseki.mod.admin.TS_FusekiServerApp; +import org.apache.jena.fuseki.mod.metrics.TestModPrometheus; +import org.apache.jena.fuseki.mod.shiro.TestModShiro; + +@Suite +@SelectClasses({ + TS_FusekiServerApp.class, + // UI + + // Prometheus + TestModPrometheus.class, + // Apache Shiro + TestModShiro.class +}) +public class TC_FusekiMods { + public TC_FusekiMods() {} +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TestFusekiServer.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TestFusekiServer.java new file mode 100644 index 00000000000..e0a6b2392df --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/TestFusekiServer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import org.junit.jupiter.api.Test; + +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.run.FusekiModServer; + +/** + * Test for the whole Fuseki server, not components. + */ +public class TestFusekiServer { + @Test public void run() { + // Setup + FusekiServer server = FusekiModServer.runAsync("--port=0", "--empty"); + int port = server.getPort(); + assertNotEquals(0, port, "Port is zero after async start"); + + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TS_FusekiServerApp.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TS_FusekiServerApp.java new file mode 100644 index 00000000000..b36360b8cba --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TS_FusekiServerApp.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import org.junit.platform.suite.api.SelectClasses; +import org.junit.platform.suite.api.Suite; + +@Suite +@SelectClasses({ + TestAdmin.class, + TestFusekiReload.class, + TestTemplateAddDataset.class, +}) +public class TS_FusekiServerApp { + +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestAdmin.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestAdmin.java new file mode 100644 index 00000000000..df0f6bafddd --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestAdmin.java @@ -0,0 +1,836 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import static org.apache.jena.fuseki.mgt.ServerMgtConst.*; +import static org.apache.jena.fuseki.server.ServerConst.opPing; +import static org.apache.jena.fuseki.server.ServerConst.opStats; +import static org.apache.jena.http.HttpOp.*; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeFalse; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.http.HttpRequest.BodyPublisher; +import java.net.http.HttpRequest.BodyPublishers; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.apache.commons.lang3.SystemUtils; +import org.apache.jena.atlas.io.IO; +import org.apache.jena.atlas.json.JSON; +import org.apache.jena.atlas.json.JsonArray; +import org.apache.jena.atlas.json.JsonObject; +import org.apache.jena.atlas.json.JsonValue; +import org.apache.jena.atlas.lib.FileOps; +import org.apache.jena.atlas.lib.Lib; +import org.apache.jena.atlas.logging.LogCtl; +import org.apache.jena.atlas.web.HttpException; +import org.apache.jena.atlas.web.TypedInputStream; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.ctl.ActionSleep; +import org.apache.jena.fuseki.ctl.JsonConstCtl; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.sys.FusekiModules; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.fuseki.mgt.ServerMgtConst; +import org.apache.jena.fuseki.server.ServerConst; +import org.apache.jena.fuseki.test.HttpTest; +import org.apache.jena.riot.WebContent; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.web.HttpSC; +import org.awaitility.Awaitility; + +/** + * Tests of the admin functionality using a pre-configured dataset + * {@link TestTemplateAddDataset}. + */ +public class TestAdmin { + + // Name of the dataset in the assembler file. + static String dsTest = "test-ds1"; + static String dsTestInf = "test-ds4"; + + // There are two Fuseki-TDB2 tests: add_delete_dataset_6() and compact_01(). + // + // On certain build systems (GH action/Linux under load, ASF Jenkins sometimes), + // add_delete_dataset_6 fails (transactions active), or compact_01 (gets a 404), + // if the two databases are the same. + static String dsTestTdb2a = "test-tdb2a"; + static String dsTestTdb2b = "test-tdb2b"; + static String fileBase = "testing/Config/"; + + private String serverURL = null; + private FusekiServer server = null; + + @BeforeEach public void startServer() { + System.setProperty("FUSEKI_BASE", "target/run"); + FileOps.clearAll("target/run"); + + server = createServerForTest(); + serverURL = server.serverURL(); + //String adminURL = server.serverURL()+"$"; + //AuthEnv.get().registerUsernamePassword(adminURL, "admin","pw"); + } + + // Exactly the module under test + private static FusekiModules moduleSetup() { + return FusekiModules.create(FMod_Admin.create()); + } + + private FusekiServer createServerForTest() { + FusekiModules modules = moduleSetup(); + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiServer testServer = FusekiServer.create() + .fusekiModules(modules) + .port(0) + .add(datasetName(), dsg) + .addServlet("/$/sleep/*", new ActionSleep()) + .build() + .start(); + return testServer; + } + + @AfterEach public void stopServer() { + if ( server != null ) + server.stop(); + serverURL = null; + // Clearup FMod_Shiro. + System.getProperties().remove(FusekiApp.envFusekiShiro); + } + + protected String urlRoot() { + return serverURL; + } + + protected String datasetName() { + return "dataset"; + } + + protected String datasetPath() { + return "/"+datasetName(); + } + + @BeforeEach public void setLogging() { + LogCtl.setLevel(Fuseki.backupLogName, "ERROR"); + LogCtl.setLevel(Fuseki.compactLogName,"ERROR"); + Awaitility.setDefaultPollDelay(20,TimeUnit.MILLISECONDS); + Awaitility.setDefaultPollInterval(50,TimeUnit.MILLISECONDS); + } + + @AfterEach public void unsetLogging() { + LogCtl.setLevel(Fuseki.backupLogName, "WARN"); + LogCtl.setLevel(Fuseki.compactLogName,"WARN"); + } + + // --- Ping + + @Test public void ping_1() { + httpGet(urlRoot()+"$/"+opPing); + } + + @Test public void ping_2() { + httpPost(urlRoot()+"$/"+opPing); + } + + // --- Server status + + @Test public void server_1() { + JsonValue jv = httpGetJson(urlRoot()+"$/"+opServer); + JsonObject obj = jv.getAsObject(); + // Now optional : assertTrue(obj.hasKey(JsonConst.admin)); + assertTrue(obj.hasKey(ServerConst.datasets)); + assertTrue(obj.hasKey(ServerMgtConst.uptime)); + assertTrue(obj.hasKey(ServerMgtConst.startDT)); + } + + @Test public void server_2() { + httpPost(urlRoot()+"$/"+opServer); + } + + // --- List all datasets + + @Test public void list_datasets_1() { + try ( TypedInputStream in = httpGet(urlRoot()+"$/"+opDatasets); ) { + IO.skipToEnd(in); + } + } + + @Test public void list_datasets_2() { + try ( TypedInputStream in = httpGet(urlRoot()+"$/"+opDatasets) ) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + JsonValue v = JSON.parseAny(in); + assertNotNull(v.getAsObject().get("datasets")); + checkJsonDatasetsAll(v); + } + } + + // Specific dataset + @Test public void list_datasets_3() { + checkExists(datasetName()); + } + + // Specific dataset + @Test public void list_datasets_4() { + HttpTest.expect404( () -> getDatasetDescription("does-not-exist") ); + } + + // Specific dataset + @Test public void list_datasets_5() { + JsonValue v = getDatasetDescription(datasetName()); + checkJsonDatasetsOne(v.getAsObject()); + } + + // Specific dataset + @Test public void add_delete_dataset_1() { + checkNotThere(dsTest); + + addTestDataset(); + + // Check exists. + checkExists(dsTest); + + // Remove it. + deleteDataset(dsTest); + checkNotThere(dsTest); + } + + // Try to add twice + @Test public void add_delete_dataset_2() { + checkNotThere(dsTest); + + try { + Path f = Path.of(fileBase+"config-ds-plain-1.ttl"); + { + httpPost(urlRoot()+"$/"+opDatasets, + WebContent.contentTypeTurtle+"; charset="+WebContent.charsetUTF8, + BodyPublishers.ofFile(f)); + } + // Check exists. + checkExists(dsTest); + try { + } catch (HttpException ex) { + httpPost(urlRoot()+"$/"+opDatasets, + WebContent.contentTypeTurtle+"; charset="+WebContent.charsetUTF8, + BodyPublishers.ofFile(f)); + assertEquals(HttpSC.CONFLICT_409, ex.getStatusCode()); + } + } catch (IOException ex) { IO.exception(ex); return; } + + // Check exists. + checkExists(dsTest); + deleteDataset(dsTest); + } + + @Test public void add_delete_dataset_3() { + checkNotThere(dsTest); + addTestDataset(); + checkExists(dsTest); + deleteDataset(dsTest); + checkNotThere(dsTest); + addTestDataset(); + checkExists(dsTest); + deleteDataset(dsTest); + } + + @Test public void add_delete_dataset_4() { + checkNotThere(dsTest); + checkNotThere(dsTestInf); + addTestDatasetInf(); + checkNotThere(dsTest); + checkExists(dsTestInf); + + deleteDataset(dsTestInf); + checkNotThere(dsTestInf); + addTestDatasetInf(); + checkExists(dsTestInf); + deleteDataset(dsTestInf); + } + + @Test public void add_delete_dataset_5() { + // New style operations : cause two fuseki:names + addTestDataset(fileBase+"config-ds-plain-2.ttl"); + checkExists("test-ds2"); + } + + @Test public void add_delete_dataset_6() { + String testDB = dsTestTdb2a; + assumeNotWindows(); + + checkNotThere(testDB); + + addTestDatasetTDB2(testDB); + + // Check exists. + checkExists(testDB); + + // Remove it. + deleteDataset(testDB); + checkNotThere(testDB); + } + + @Test public void add_error_1() { + HttpTest.execWithHttpException(HttpSC.BAD_REQUEST_400, + ()-> addTestDataset(fileBase+"config-ds-bad-name-1.ttl")); + } + + @Test public void add_error_2() { + HttpTest.execWithHttpException(HttpSC.BAD_REQUEST_400, + ()-> addTestDataset(fileBase+"config-ds-bad-name-2.ttl")); + } + + @Test public void add_error_3() { + HttpTest.execWithHttpException(HttpSC.BAD_REQUEST_400, + ()-> addTestDataset(fileBase+"config-ds-bad-name-3.ttl")); + } + + @Test public void add_error_4() { + HttpTest.execWithHttpException(HttpSC.BAD_REQUEST_400, + ()-> addTestDataset(fileBase+"config-ds-bad-name-4.ttl")); + } + + @Test public void delete_dataset_1() { + String name = "NoSuchDataset"; + HttpTest.expect404( ()-> httpDelete(urlRoot()+"$/"+opDatasets+"/"+name) ); + } + + // ---- Backup + + @Test public void create_backup_1() { + String id = null; + try { + JsonValue v = httpPostRtnJSON(urlRoot() + "$/" + opBackup + "/" + datasetName()); + id = v.getAsObject().getString("taskId"); + } finally { + waitForTasksToFinish(1000, 10, 20000); + } + assertNotNull(id); + checkInTasks(id); + + // Check a backup was created + try ( TypedInputStream in = httpGet(urlRoot()+"$/"+opListBackups) ) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + JsonValue v = JSON.parseAny(in); + assertNotNull(v.getAsObject().get("backups")); + JsonArray a = v.getAsObject().get("backups").getAsArray(); + assertEquals(1, a.size()); + } + + JsonValue task = getTask(id); + assertNotNull(id); + // Expect task success + assertTrue(task.getAsObject().getBoolean(JsonConstCtl.success), "Expected task to be marked as successful"); + } + + @Test + public void create_backup_2() { + HttpTest.expect400(()->{ + JsonValue v = httpPostRtnJSON(urlRoot() + "$/" + opBackup + "/noSuchDataset"); + }); + } + + @Test public void list_backups_1() { + try ( TypedInputStream in = httpGet(urlRoot()+"$/"+opListBackups) ) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + JsonValue v = JSON.parseAny(in); + assertNotNull(v.getAsObject().get("backups")); + } + } + + // ---- Compact + + @Test public void compact_01() { + assumeNotWindows(); + + String testDB = dsTestTdb2b; + try { + checkNotThere(testDB); + addTestDatasetTDB2(testDB); + checkExists(testDB); + + String id = null; + try { + JsonValue v = httpPostRtnJSON(urlRoot() + "$/" + opCompact + "/" + testDB); + id = v.getAsObject().getString(JsonConstCtl.taskId); + } finally { + waitForTasksToFinish(1000, 500, 20_000); + } + assertNotNull(id); + checkInTasks(id); + + JsonValue task = getTask(id); + // ---- + // The result assertion is throwing NPE occasionally on some heavily loaded CI servers. + // This may be because of server or test code encountering a very long wait. + // These next statements check the assumed structure of the return. + assertNotNull(task, "Task value"); + JsonObject obj = task.getAsObject(); + assertNotNull(obj, "Task.getAsObject()"); + // Provoke code to get a stacktrace. + obj.getBoolean(JsonConstCtl.success); + // ---- + // The assertion we really wanted to check. + // Check task success + assertTrue(task.getAsObject().getBoolean(JsonConstCtl.success), + "Expected task to be marked as successful"); + } finally { + deleteDataset(testDB); + } + } + + @Test public void compact_02() { + HttpTest.expect400(()->{ + JsonValue v = httpPostRtnJSON(urlRoot() + "$/" + opCompact + "/noSuchDataset"); + }); + } + + private void assumeNotWindows() { + assumeFalse(SystemUtils.IS_OS_WINDOWS, "Test may be unstable on Windows due to inability to delete memory-mapped files"); + } + + // ---- Server + + // ---- Stats + + @Test public void stats_1() { + JsonValue v = execGetJSON(urlRoot()+"$/"+opStats); + checkJsonStatsAll(v); + } + + @Test public void stats_2() { + addTestDataset(); + JsonValue v = execGetJSON(urlRoot()+"$/"+opStats+datasetPath()); + checkJsonStatsAll(v); + deleteDataset(dsTest); + } + + @Test public void stats_3() { + addTestDataset(); + HttpTest.expect404(()-> execGetJSON(urlRoot()+"$/"+opStats+"/DoesNotExist")); + deleteDataset(dsTest); + } + + @Test public void stats_4() { + JsonValue v = execPostJSON(urlRoot()+"$/"+opStats); + checkJsonStatsAll(v); + } + + @Test public void stats_5() { + addTestDataset(); + JsonValue v = execPostJSON(urlRoot()+"$/"+opStats+datasetPath()); + checkJsonStatsAll(v); + deleteDataset(dsTest); + } + + @Test public void sleep_1() { + String x = execSleepTask(null, 1); + } + + @Test public void sleep_2() { + try { + String x = execSleepTask(null, -1); + fail("Sleep call unexpectedly succeed"); + } catch (HttpException ex) { + assertEquals(400, ex.getStatusCode()); + } + } + + @Test public void sleep_3() { + try { + String x = execSleepTask(null, 20*1000+1); + fail("Sleep call unexpectedly succeed"); + } catch (HttpException ex) { + assertEquals(400, ex.getStatusCode()); + } + } + + // Async task testing + + @Test public void task_1() { + String x = execSleepTask(null, 10); + assertNotNull(x); + Integer.parseInt(x); + } + + @Test public void task_2() { + String x = "NoSuchTask"; + String url = urlRoot()+"$/tasks/"+x; + HttpTest.expect404(()->httpGetJson(url) ); + try { + checkInTasks(x); + fail("No failure!"); + } catch (AssertionError ex) {} + } + + + @Test public void task_3() { + // Timing dependent. + // Create a "long" running task so we can find it. + String x = execSleepTask(null, 100); + checkTask(x); + checkInTasks(x); + assertNotNull(x); + Integer.parseInt(x); + } + + @Test public void task_4() { + // Timing dependent. + // Create a "short" running task + String x = execSleepTask(null, 1); + // Check exists in the list of all tasks (should be "finished") + checkInTasks(x); + String url = urlRoot()+"$/tasks/"+x; + + boolean finished = false; + for ( int i = 0; i < 10; i++ ) { + if ( i != 0 ) + Lib.sleep(25); + JsonValue v = httpGetJson(url); + checkTask(v); + if ( v.getAsObject().hasKey("finished") ) { + finished = true; + break; + } + } + if ( ! finished ) + fail("Task has not finished"); + } + + @Test public void task_5() { + // Short running task - still in info API call. + String x = execSleepTask(null, 1); + checkInTasks(x); + } + + @Test public void task_6() { + String x1 = execSleepTask(null, 1000); + String x2 = execSleepTask(null, 1000); + await().timeout(500,TimeUnit.MILLISECONDS).until(() -> runningTasks().size() > 1); + await().timeout(2000, TimeUnit.MILLISECONDS).until(() -> runningTasks().isEmpty()); + } + + @Test public void task_7() { + try { + String x1 = execSleepTask(null, 1000); + String x2 = execSleepTask(null, 1000); + String x3 = execSleepTask(null, 1000); + String x4 = execSleepTask(null, 1000); + try { + // Try to make test more stable on a loaded CI server. + // Unloaded the first sleep will fail but due to slowness/burstiness + // some tasks above may have completed. + String x5 = execSleepTask(null, 4000); + String x6 = execSleepTask(null, 4000); + String x7 = execSleepTask(null, 4000); + String x8 = execSleepTask(null, 10); + fail("Managed to add a 5th test"); + } catch (HttpException ex) { + assertEquals(HttpSC.BAD_REQUEST_400, ex.getStatusCode()); + } + } finally { + waitForTasksToFinish(1000, 250, 4000); + } + } + + /** Expect two string to be non-null and be {@link String#equalsIgnoreCase} */ + private void assertEqualsIgnoreCase(String expected, String actual) { + if ( expected == null && actual == null ) + return; + if ( expected == null || actual == null ) + fail("Expected: "+expected+" Got: "+actual); + if ( ! expected.equalsIgnoreCase(actual) ) + fail("Expected: "+expected+" Got: "+actual); + } + + private JsonValue getTask(String taskId) { + String url = urlRoot()+"$/tasks/"+taskId; + return httpGetJson(url); + } + + private JsonValue getDatasetDescription(String dsName) { + if ( dsName.startsWith("/") ) + dsName = dsName.substring(1); + try (TypedInputStream in = httpGet(urlRoot() + "$/" + opDatasets + "/" + dsName)) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + JsonValue v = JSON.parse(in); + return v; + } + } + + // -- Add + + private void addTestDataset() { + addTestDataset(fileBase+"config-ds-plain-1.ttl"); + } + + private void addTestDatasetInf() { + addTestDataset(fileBase+"config-ds-inf.ttl"); + } + + private void addTestDatasetTDB2(String DBname) { + Objects.nonNull(DBname); + if ( DBname.equals(dsTestTdb2a) ) { + addTestDataset(fileBase+"config-tdb2a.ttl"); + return; + } + if ( DBname.equals(dsTestTdb2b) ) { + addTestDataset(fileBase+"config-tdb2b.ttl"); + return; + } + throw new IllegalArgumentException("No configuration for "+DBname); + } + + private void addTestDataset(String filename) { + try { + Path f = Path.of(filename); + BodyPublisher body = BodyPublishers.ofFile(f); + String ct = WebContent.contentTypeTurtle; + httpPost(urlRoot()+"$/"+opDatasets, ct, body); + } catch (FileNotFoundException e) { + IO.exception(e); + } + } + + private void deleteDataset(String name) { + httpDelete(urlRoot()+"$/"+opDatasets+"/"+name); + } + + private String execSleepTask(String name, int millis) { + String url = urlRoot()+"$/sleep"; + if ( name != null ) { + if ( name.startsWith("/") ) + name = name.substring(1); + url = url + "/"+name; + } + + JsonValue v = httpPostRtnJSON(url+"?interval="+millis); + String id = v.getAsObject().getString("taskId"); + return id; + } + + private void checkTask(String x) { + String url = urlRoot()+"$/tasks/"+x; + JsonValue v = httpGetJson(url); + checkTask(v); + } + + private void checkTask(JsonValue v) { + assertNotNull(v); + assertTrue(v.isObject()); + //System.out.println(v); + JsonObject obj = v.getAsObject(); + try { + assertTrue(obj.hasKey("task")); + assertTrue(obj.hasKey("taskId")); + // Not present until it runs : "started" + } catch (AssertionError ex) { + System.out.println(obj); + throw ex; + } + } + + private void checkInTasks(String x) { + String url = urlRoot()+"$/tasks"; + JsonValue v = httpGetJson(url); + assertTrue(v.isArray()); + JsonArray array = v.getAsArray(); + int found = 0; + for ( int i = 0; i < array.size(); i++ ) { + JsonValue jv = array.get(i); + assertTrue(jv.isObject()); + JsonObject obj = jv.getAsObject(); + checkTask(obj); + if ( obj.getString("taskId").equals(x) ) { + found++; + } + } + assertEquals(1, found, "Occurrence of taskId count"); + } + + private List runningTasks(String... x) { + String url = urlRoot()+"$/tasks"; + JsonValue v = httpGetJson(url); + assertTrue(v.isArray()); + JsonArray array = v.getAsArray(); + List running = new ArrayList<>(); + for ( int i = 0; i < array.size(); i++ ) { + JsonValue jv = array.get(i); + assertTrue(jv.isObject()); + JsonObject obj = jv.getAsObject(); + if ( isRunning(obj) ) + running.add(obj.getString("taskId")); + } + return running; + } + + /** + * Wait for tasks to all finish. + * Algorithm: wait for {@code pause}, then start polling for upto {@code maxWaitMillis}. + * Intervals in milliseconds. + * @param pauseMillis + * @param pollInterval + * @param maxWaitMillis + * @return + */ + private boolean waitForTasksToFinish(int pauseMillis, int pollInterval, int maxWaitMillis) { + // Wait for them to finish. + // Divide into chunks + if ( pauseMillis > 0 ) + Lib.sleep(pauseMillis); + long start = System.currentTimeMillis(); + long endTime = start + maxWaitMillis; + final int intervals = maxWaitMillis/pollInterval; + long now = start; + for (int i = 0 ; i < intervals ; i++ ) { + // May have waited (much) longer than the pollInterval : heavily loaded build systems. + if ( now-start > maxWaitMillis ) + break; + List x = runningTasks(); + if ( x.isEmpty() ) + return true; + Lib.sleep(pollInterval); + now = System.currentTimeMillis(); + } + return false; + } + + private boolean isRunning(JsonObject taskObj) { + checkTask(taskObj); + return taskObj.hasKey("started") && ! taskObj.hasKey("finished"); + } + + private void askPing(String name) { + if ( name.startsWith("/") ) + name = name.substring(1); + try ( TypedInputStream in = httpGet(urlRoot()+name+"/sparql?query=ASK%7B%7D") ) { + IO.skipToEnd(in); + } + } + + private void adminPing(String name) { + try ( TypedInputStream in = httpGet(urlRoot()+"$/"+opDatasets+"/"+name) ) { + IO.skipToEnd(in); + } + } + + private void checkExists(String name) { + adminPing(name); + askPing(name); + } + + private void checkExistsNotActive(String name) { + adminPing(name); + try { askPing(name); + fail("askPing did not cause an Http Exception"); + } catch ( HttpException ex ) {} + JsonValue v = getDatasetDescription(name); + assertFalse(v.getAsObject().get("ds.state").getAsBoolean().value()); + } + + private void checkNotThere(String name) { + String n = (name.startsWith("/")) ? name.substring(1) : name; + // Check gone exists. + HttpTest.expect404(()-> adminPing(n) ); + HttpTest.expect404(() -> askPing(n) ); + } + + private void checkJsonDatasetsAll(JsonValue v) { + assertNotNull(v.getAsObject().get("datasets")); + JsonArray a = v.getAsObject().get("datasets").getAsArray(); + for ( JsonValue v2 : a ) + checkJsonDatasetsOne(v2); + } + + private void checkJsonDatasetsOne(JsonValue v) { + assertTrue(v.isObject()); + JsonObject obj = v.getAsObject(); + assertNotNull(obj.get("ds.name")); + assertNotNull(obj.get("ds.services")); + assertNotNull(obj.get("ds.state")); + assertTrue(obj.get("ds.services").isArray()); + } + + private void checkJsonStatsAll(JsonValue v) { + assertNotNull(v.getAsObject().get("datasets")); + JsonObject a = v.getAsObject().get("datasets").getAsObject(); + for ( String dsname : a.keys() ) { + JsonValue obj = a.get(dsname).getAsObject(); + checkJsonStatsOne(obj); + } + } + + private void checkJsonStatsOne(JsonValue v) { + checkJsonStatsCounters(v); + JsonObject obj1 = v.getAsObject().get("endpoints").getAsObject(); + for ( String srvName : obj1.keys() ) { + JsonObject obj2 = obj1.get(srvName).getAsObject(); + assertTrue(obj2.hasKey("description")); + assertTrue(obj2.hasKey("operation")); + checkJsonStatsCounters(obj2); + } + } + + private void checkJsonStatsCounters(JsonValue v) { + JsonObject obj = v.getAsObject(); + assertTrue(obj.hasKey("Requests")); + assertTrue(obj.hasKey("RequestsGood")); + assertTrue(obj.hasKey("RequestsBad")); + } + + private JsonValue execGetJSON(String url) { + try ( TypedInputStream in = httpGet(url) ) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + return JSON.parse(in); + } + } + + private JsonValue execPostJSON(String url) { + try ( TypedInputStream in = httpPostStream(url, null, null, null) ) { + assertEqualsIgnoreCase(WebContent.contentTypeJSON, in.getContentType()); + return JSON.parse(in); + } + } + + /* + GET /$/ping + POST /$/ping + POST /$/datasets/ + GET /$/datasets/ + DELETE /$/datasets/*{name}* + GET /$/datasets/*{name}* + POST /$/datasets/*{name}*?state=offline + POST /$/datasets/*{name}*?state=active + POST /$/backup/*{name}* + POST /$/compact/*{name}* + GET /$/server + POST /$/server/shutdown + GET /$/stats/ + GET /$/stats/*{name}* + */ +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestFusekiReload.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestFusekiReload.java new file mode 100644 index 00000000000..ee68e96df1b --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestFusekiReload.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; + +import org.apache.jena.atlas.io.IOX; +import org.apache.jena.atlas.logging.LogCtl; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.FusekiTestLib; +import org.apache.jena.fuseki.mgt.ActionReload; +import org.apache.jena.http.HttpOp; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.riot.RDFParser; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.engine.http.QueryExceptionHTTP; +import org.apache.jena.sparql.exec.QueryExec; +import org.apache.jena.sparql.exec.RowSetOps; +import org.apache.jena.sparql.exec.http.QueryExecHTTP; + +public class TestFusekiReload { + + private static Path fConfigServer = Path.of("target/config-reload.ttl"); + private static Path DIR = Path.of("testing/Config/"); + private static Path fConfig1 = DIR.resolve("reload-config1.ttl"); + private static Path fConfig2 = DIR.resolve("reload-config2.ttl"); + + @Before public void before() { + // Initial state + copyFile(fConfig1, fConfigServer); + } + + @AfterClass public static void after() { + try { + Files.delete(fConfigServer); + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } + + + @Test public void serverReload_1() { + FusekiServer server = server(fConfigServer); + try { + server.start(); + + serverBeforeReload(server); + + // Change configuration file. + copyFile(fConfig2, fConfigServer); + Model newConfig = RDFParser.source(fConfigServer).toModel(); + + // Reload operation on the server + HttpOp.httpPost("http://localhost:"+server.getPort()+"/$/reload"); + + serverAfterReload(server); + } + finally { server.stop(); } + } + + @Test public void serverReload_2() { + FusekiServer server = serverNoConfigFile(); + try { + server.start(); + + serverBeforeReload(server); + + // Change configuration file. + // Error! + copyFile(fConfig2, fConfigServer); + Model newConfig = RDFParser.source(fConfigServer).toModel(); + + LogCtl.withLevel(Fuseki.serverLog, "ERROR", ()-> + // Poke server - Operation denied - no configuration file. + FusekiTestLib.expect400(()->HttpOp.httpPost("http://localhost:"+server.getPort()+"/$/reload")) + ); + + // No change + serverBeforeReload(server); + } + finally { server.stop(); } + } + + private static void copyFile(Path pathSrc, Path pathDest) { + try { + Files.copy(pathSrc, pathDest, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { throw IOX.exception(ex); } + } + + private static void serverBeforeReload(FusekiServer server) { + String URL = "http://localhost:"+server.getPort()+"/$/ping"; + String x = HttpOp.httpGetString(URL); + query(server, "/ds", "SELECT * { }", 200); + query(server, "/dataset2", "SELECT * { ?s ?p ?o }", 404); + query(server, "/zero", "SELECT * { }", 404); + query(server, "/codedsg", "SELECT * { }", 200); + } + + private static void serverAfterReload(FusekiServer server) { + String URL = "http://localhost:"+server.getPort()+"/$/ping"; + String x = HttpOp.httpGetString(URL); + query(server, "/ds", "SELECT * { }", 404); + query(server, "/dataset2", "SELECT * { ?s ?p ?o }", 200); + query(server, "/zero", "SELECT * { }", 404); + // Replaced. + query(server, "/codedsg", "SELECT * { }", 404); + } + + private static void query(FusekiServer server, String datasetName, String queryString, int expectedStatusCode) { + QueryExec qExec = QueryExecHTTP.service(server.datasetURL(datasetName)).query(queryString).build(); + try { + RowSetOps.consume(qExec.select()); + assertEquals(datasetName, expectedStatusCode, 200); + } catch (QueryExceptionHTTP ex) { + assertEquals(datasetName, expectedStatusCode, ex.getStatusCode()); + } + } + + private static FusekiServer serverNoConfigFile() { + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiServer server = FusekiServer.create().port(0) + // .verbose(true) + .addServlet("/$/reload", new ActionReload()) + .add("/ds", dsg) + .add("/codedsg", dsg) + .build(); + return server; + } + + + private static FusekiServer server(Path fConfig) { + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiServer server = FusekiServer.create().port(0) + // .verbose(true) + .addServlet("/$/reload", new ActionReload()) + .parseConfigFile(fConfig) + .add("/codedsg", dsg) + .build(); + return server; + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestTemplateAddDataset.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestTemplateAddDataset.java new file mode 100644 index 00000000000..975876095dc --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/admin/TestTemplateAddDataset.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.admin; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.*; + +import org.apache.jena.atlas.lib.FileOps; +import org.apache.jena.atlas.logging.LogCtl; +import org.apache.jena.atlas.web.HttpException; +import org.apache.jena.atlas.web.TypedInputStream; +import org.apache.jena.fuseki.Fuseki; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.sys.FusekiModules; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.http.HttpOp; +import org.apache.jena.query.QueryExecution; +import org.apache.jena.rdfconnection.RDFConnection; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.exec.http.Params; +import org.apache.jena.web.HttpSC; +import org.awaitility.Awaitility; + +/** + * Tests of the admin functionality on an empty server and using the template mechanism. + * See also {@link TestAdmin}. + */ +public class TestTemplateAddDataset { + + // One server for all tests + private static String serverURL = null; + private static FusekiServer server = null; + + @BeforeAll public static void startServer() { + System.setProperty("FUSEKI_BASE", "target/run"); + FileOps.clearAll("target/run"); + + server = createServerForTest(); + serverURL = server.serverURL(); + } + + // Exactly the module under test + private static FusekiModules moduleSetup() { + return FusekiModules.create(FMod_Admin.create()); + } + + private static FusekiServer createServerForTest() { + FusekiModules modules = moduleSetup(); + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiServer testServer = FusekiServer.create() + .fusekiModules(modules) + .port(0) + .build() + .start(); + return testServer; + } + + @AfterAll public static void stopServer() { + if ( server != null ) + server.stop(); + serverURL = null; + // Clearup FMod_Shiro. + System.getProperties().remove(FusekiApp.envFusekiShiro); + } + + protected String urlRoot() { + return serverURL; + } + + protected String adminURL() { + return serverURL+"$/"; + } + + @BeforeEach public void setLogging() { + LogCtl.setLevel(Fuseki.backupLogName, "ERROR"); + LogCtl.setLevel(Fuseki.compactLogName,"ERROR"); + Awaitility.setDefaultPollDelay(20,TimeUnit.MILLISECONDS); + Awaitility.setDefaultPollInterval(50,TimeUnit.MILLISECONDS); + } + + @AfterEach public void unsetLogging() { + LogCtl.setLevel(Fuseki.backupLogName, "WARN"); + LogCtl.setLevel(Fuseki.compactLogName,"WARN"); + } + + @Order(value = 1) + @Test public void add_delete_api_1() throws Exception { + if ( org.apache.jena.tdb1.sys.SystemTDB.isWindows ) + return; + testAddDelete("db_mem", "mem", false, false); + } + + @Order(value = 2) + @Test public void add_delete_api_2() throws Exception { + if ( org.apache.jena.tdb1.sys.SystemTDB.isWindows ) + return; + // This should fail. + HttpException ex = assertThrows(HttpException.class, ()->testAddDelete("db_mem", "mem", true, false)); + // 409 conflict - "a request conflicts with the current state of the target resource." + // and the target resource is the container "/$/datasets" + assertEquals(HttpSC.CONFLICT_409, ex.getStatusCode()); + } + + private void testAddDelete(String dbName, String dbType, boolean alreadyExists, boolean hasFiles) { + String datasetURL = server.datasetURL(dbName); + Params params = Params.create().add("dbName", dbName).add("dbType", dbType); + + if ( alreadyExists ) + assertTrue(exists(datasetURL)); + else + assertFalse(exists(datasetURL)); + + // Use the template + HttpOp.httpPostForm(adminURL()+"datasets", params); + + RDFConnection conn = RDFConnection.connect(server.datasetURL(dbName)); + conn.update("INSERT DATA { 123 }"); + int x1 = count(conn); + assertEquals(1, x1); + + Path pathDB = FusekiApp.dirDatabases.resolve(dbName); + + if ( hasFiles ) + assertTrue(Files.exists(pathDB)); + + HttpOp.httpDelete(adminURL()+"datasets/"+dbName); + + assertFalse(exists(datasetURL)); + + //if ( hasFiles ) + assertFalse(Files.exists(pathDB)); + + // Recreate : no contents. + HttpOp.httpPostForm(adminURL()+"datasets", params); + assertTrue(exists(datasetURL), ()->"false: exists("+datasetURL+")"); + int x2 = count(conn); + assertEquals(0, x2); + if ( hasFiles ) + assertTrue(Files.exists(pathDB)); + } + + private static boolean exists(String url) { + try ( TypedInputStream in = HttpOp.httpGet(url) ) { + return true; + } catch (HttpException ex) { + if ( ex.getStatusCode() == HttpSC.NOT_FOUND_404 ) + return false; + throw ex; + } + } + + static int count(RDFConnection conn) { + try ( QueryExecution qExec = conn.query("SELECT (count(*) AS ?C) { ?s ?p ?o }")) { + return qExec.execSelect().next().getLiteral("C").getInt(); + } + } +} + diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/metrics/TestModPrometheus.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/metrics/TestModPrometheus.java new file mode 100644 index 00000000000..ce3be524fb9 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/metrics/TestModPrometheus.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.metrics; + +import static org.apache.jena.http.HttpLib.handleResponseRtnString; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.InputStream; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.sys.FusekiModules; +import org.apache.jena.fuseki.mod.prometheus.FMod_Prometheus; +import org.apache.jena.http.HttpEnv; +import org.apache.jena.http.HttpLib; +import org.apache.jena.riot.WebContent; +import org.apache.jena.riot.web.HttpNames; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; + +public class TestModPrometheus { + + private FusekiServer testServer = null; + + @BeforeEach void setupServer() { + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiModules fusekiModules = FusekiModules.create(FMod_Prometheus.get()); + testServer = FusekiServer.create() + .add("/ds", dsg) + .enableMetrics(false) // N.B. false. Instead, use module to setup. + .fusekiModules(fusekiModules) + .build(); + testServer.start(); + } + + @AfterEach void teardownServer() { + if ( testServer != null ) + testServer.stop(); + } + + @Test + public void can_retrieve_metrics() { + String metricsURL = testServer.serverURL()+"$/metrics"; + HttpRequest request = HttpRequest.newBuilder().uri(HttpLib.toRequestURI(metricsURL)).build(); + HttpResponse response = HttpLib.executeJDK(HttpEnv.getDftHttpClient(), request, BodyHandlers.ofInputStream()); + String body = handleResponseRtnString(response); + + String ct = response.headers().firstValue(HttpNames.hContentType).orElse(null); + assertNotNull(ct, "No Content-Type"); + assertTrue(ct.contains(WebContent.contentTypeTextPlain)); + assertTrue(ct.contains(WebContent.charsetUTF8)); + assertTrue(body.contains("fuseki_requests_good")); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/shiro/TestModShiro.java b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/shiro/TestModShiro.java new file mode 100644 index 00000000000..dafc636f78d --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/src/test/java/org/apache/jena/fuseki/mod/shiro/TestModShiro.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.jena.fuseki.mod.shiro; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.Authenticator; +import java.net.http.HttpClient; +import java.util.regex.Pattern; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import org.apache.jena.atlas.net.Host; +import org.apache.jena.atlas.web.HttpException; +import org.apache.jena.fuseki.main.FusekiServer; +import org.apache.jena.fuseki.main.cmds.FusekiMain; +import org.apache.jena.fuseki.main.sys.FusekiModule; +import org.apache.jena.fuseki.main.sys.FusekiModules; +import org.apache.jena.fuseki.mgt.FusekiApp; +import org.apache.jena.fuseki.system.FusekiLogging; +import org.apache.jena.graph.Graph; +import org.apache.jena.http.HttpEnv; +import org.apache.jena.http.HttpOp; +import org.apache.jena.http.auth.AuthEnv; +import org.apache.jena.http.auth.AuthLib; +import org.apache.jena.sparql.core.DatasetGraph; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.exec.http.GSP; +import org.apache.jena.sparql.exec.http.QueryExecHTTP; + +public class TestModShiro { + static final String unlocal = Host.getHostAddress(); + static final String localRE = Pattern.quote("localhost"); + + static { + FusekiLogging.setLogging(); + // Incase it is finding file:log4j.properties first. +// LogCtl.disable(Fuseki.serverLog); +// LogCtl.disable(Fuseki.actionLog); +// LogCtl.disable(FMod_Shiro.shiroConfigLog); + } + + @BeforeEach void before() { + System.getProperties().remove(FusekiApp.envFusekiShiro); + AuthEnv.get().clearAuthEnv(); + } + + @AfterEach void after() { + AuthEnv.get().clearAuthEnv(); + } + + @AfterAll static void afterAll() { + System.getProperties().remove(FusekiApp.envFusekiShiro); + } + + private String unlocalhost(FusekiServer server, String dataset) { + String local = server.datasetURL(dataset); + if ( unlocal != null ) + local = local.replaceFirst(localRE, unlocal); + return local; + } + + /** Builder for a server with Shiro */ + private FusekiServer.Builder serverBuilderWithShiro(String filename) { + System.getProperties().setProperty(FusekiApp.envFusekiShiro, filename); + FusekiModules modules = FusekiModules.create(FMod_Shiro.create()); + return FusekiServer.create() + .port(0) + .fusekiModules(modules); + } + + @Test public void access_localhost() { + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiModules modules = FusekiModules.create(FMod_Shiro.create()); + FusekiServer server = serverBuilderWithShiro("testing/Shiro/shiro_localhost.ini") + .add("/local/ds", dsg) + .add("/public/ds", dsg) + .build(); + server.start(); + String dsPublic = "/public/ds"; + String dsLocal = "/local/ds"; + try { + attemptByAddr(server, dsPublic); + HttpException httpEx = assertThrows(HttpException.class, ()->attemptByAddr(server, dsLocal)); + assertEquals(403, httpEx.getStatusCode(), "Expected HTTP 403"); + + attemptByLocalhost(server, dsLocal); + } finally { + server.stop(); + AuthEnv.get().clearAuthEnv(); + } + } + + @Test public void access_userPassword() { + String dsname = "/ds"; + DatasetGraph dsg = DatasetGraphFactory.createTxnMem(); + FusekiServer server = serverBuilderWithShiro("testing/Shiro/shiro_userpassword.ini") + .add(dsname, dsg) + .enablePing(true) + .build(); + server.start(); + + String URL = server.datasetURL(dsname); + + try { + // No user-password + { + HttpException httpEx = assertThrows(HttpException.class, ()->attemptByLocalhost(server, dsname)); + assertEquals(401, httpEx.getStatusCode(), "Expected HTTP 401"); + } + + // user-password via authenticator: localhost + { + Authenticator authenticator = AuthLib.authenticator("user1", "passwd1"); + HttpClient httpClient = HttpEnv.httpClientBuilder().authenticator(authenticator).build(); + attemptByLocalhost(server, httpClient, dsname); + // and a SPARQL query + QueryExecHTTP.service(URL).httpClient(httpClient).query("ASK{}").ask(); + } + + // user-password via registration + { + AuthEnv.get().registerUsernamePassword(server.serverURL(), "user1", "passwd1"); + attemptByLocalhost(server, dsname); + AuthEnv.get().unregisterUsernamePassword(server.serverURL()); + } + + // try the ping (proxy for admin operations) + { + Authenticator authenticator = AuthLib.authenticator("admin", "pw"); + HttpClient httpClient = HttpEnv.httpClientBuilder().authenticator(authenticator).build(); + HttpOp.httpGetString(httpClient, server.serverURL()+"$/ping"); + AuthEnv.get().unregisterUsernamePassword(server.serverURL()); + } + + { + // Bad password + AuthEnv.get().registerUsernamePassword(server.serverURL(), "user1", "passwd2"); + HttpException httpEx = assertThrows(HttpException.class, ()->attemptByLocalhost(server, dsname)); + assertEquals(401, httpEx.getStatusCode(), "Expected HTTP 401"); + AuthEnv.get().unregisterUsernamePassword(server.serverURL()); + } + + } finally { + server.stop(); + AuthEnv.get().clearAuthEnv(); + } + } + + @Test public void shiroByCommandLine() { + String dsname = "/ds"; + FusekiModule fmod = FMod_Shiro.create(); + FusekiMain.addCustomiser(fmod); + + // And also a module! + FusekiServer server = FusekiMain.builder("--port=0", "--shiro=testing/Shiro/shiro_userpassword.ini", "--mem", dsname) + // Must be same instance. + .fusekiModules(FusekiModules.create(fmod)) + .build(); + server.start(); + try { + // No user-password + HttpException httpEx = assertThrows(HttpException.class, ()->attemptByLocalhost(server, dsname)); + assertEquals(401, httpEx.getStatusCode(), "Expected HTTP 401"); + } finally { server.stop(); } + FusekiMain.resetCustomisers(); + } + + // ---------------------------- + + // TEST **** command line + + private void attemptByAddr(FusekiServer server, String dsname) { + attemptByAddr(server, null, dsname); + } + + private void attemptByAddr(FusekiServer server, HttpClient httpClient, String dsname) { + String URL = server.datasetURL(dsname); + String URLip = unlocalhost(server, dsname); + attempt(URLip, httpClient); + } + + private void attemptByLocalhost(FusekiServer server, String dsname) { + attemptByLocalhost(server, null, dsname); + } + + private void attemptByLocalhost(FusekiServer server, HttpClient httpClient, String dsname) { + String URL = server.datasetURL(dsname); + attempt(URL, httpClient); + } + + private void attempt(String URL, HttpClient httpClient) { + GSP gsp = GSP.service(URL).defaultGraph(); + if ( httpClient != null ) + gsp.httpClient(httpClient); + Graph g = gsp.GET(); + assertNotNull(g); + } +} diff --git a/jena-fuseki2/jena-fuseki-main/src/test/resources/log4j2-test.properties b/jena-fuseki2/jena-fuseki-main/src/test/resources/log4j2-test.properties index ac4c56bbcd6..26b8c4b2e2d 100644 --- a/jena-fuseki2/jena-fuseki-main/src/test/resources/log4j2-test.properties +++ b/jena-fuseki2/jena-fuseki-main/src/test/resources/log4j2-test.properties @@ -25,18 +25,8 @@ logger.arq-exec.level = INFO logger.fuseki.name = org.apache.jena.fuseki logger.fuseki.level = WARN -## Some tests generate request WARNings. logger.fuseki-fuseki.name = org.apache.jena.fuseki.Fuseki -logger.fuseki-fuseki.level = ERROR - -#logger.fuseki-server.name = org.apache.jena.fuseki.Server -#logger.fuseki-server.level = INFO -# -#logger.fuseki-admin.name = org.apache.jena.fuseki.Admin -#logger.fuseki-admin.level = INFO -# -#logger.fuseki-compact.name = org.apache.jena.fuseki.Compact -#logger.fuseki-compact.level = WARN +logger.fuseki-fuseki.level = WARN logger.fuseki-autoload.name = org.apache.jena.fuseki.main.sys.FusekiAutoModules logger.fuseki-autoload.level = ERROR @@ -47,6 +37,9 @@ logger.http.level = INFO logger.riot.name = org.apache.jena.riot logger.riot.level = INFO +logger.riot.name = org.apache.shiro +logger.riot.level = WARN + logger.jetty.name = org.eclipse.jetty logger.jetty.level = WARN diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-1.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-1.ttl new file mode 100644 index 00000000000..05640fd0cf8 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-1.ttl @@ -0,0 +1,15 @@ +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . + +@prefix rdfs: . +@prefix ja: . + +<#service1> rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/ds + fuseki:name "" ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:RDFDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-2.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-2.ttl new file mode 100644 index 00000000000..27df06bcb45 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-2.ttl @@ -0,0 +1,15 @@ +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . + +@prefix rdfs: . +@prefix ja: . + +<#service1> rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/ds + fuseki:name " " ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:RDFDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-3.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-3.ttl new file mode 100644 index 00000000000..19caef108be --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-3.ttl @@ -0,0 +1,15 @@ +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . + +@prefix rdfs: . +@prefix ja: . + +<#service1> rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/ds + fuseki:name "ABC DEF" ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:RDFDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-4.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-4.ttl new file mode 100644 index 00000000000..f57840e4185 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-bad-name-4.ttl @@ -0,0 +1,15 @@ +@prefix : <#> . +@prefix fuseki: . +@prefix rdf: . + +@prefix rdfs: . +@prefix ja: . + +<#service1> rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/ds + fuseki:name "/ABC DEF " ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:RDFDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-inf.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-inf.ttl new file mode 100644 index 00000000000..b75cf18ce38 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-inf.ttl @@ -0,0 +1,30 @@ +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: + +PREFIX rdfs: +PREFIX ja: +PREFIX tdb2: + +<#service2> rdf:type fuseki:Service ; + fuseki:name "test-ds4" ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#infDataset> ; + . + +<#infDataset> rdf:type ja:RDFDataset ; + ja:defaultGraph <#infGraph> ; + . + +<#infGraph> a ja:InfModel; + ja:reasoner [ja:reasonerURL ] ; + ja:baseModel <#tdb_graph> ; + . + +<#tdb_graph> rdf:type tdb2:GraphTDB ; + tdb2:dataset <#tdb_dataset> ; + . + +<#tdb_dataset> rdf:type tdb2:DatasetTDB ; + tdb2:location "--mem--" ; + . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-1.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-1.ttl new file mode 100644 index 00000000000..f993f350fd6 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-1.ttl @@ -0,0 +1,17 @@ +## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: + +PREFIX rdfs: +PREFIX ja: + +<#service1> #rdf:type fuseki:Service ; + # URI of the dataset -- http://host:port/ds + fuseki:name "test-ds1" ; + fuseki:serviceQuery "sparql" ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:RDFDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-2.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-2.ttl new file mode 100644 index 00000000000..d4b3fc90221 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-ds-plain-2.ttl @@ -0,0 +1,18 @@ +## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: +PREFIX rdfs: +PREFIX ja: + +<#service1> rdf:type fuseki:Service ; + fuseki:name "test-ds2" ; + fuseki:endpoint [ + fuseki:operation fuseki:query ; + fuseki:name "sparql" + ] ; + fuseki:dataset <#emptyDataset> ; + . + +<#emptyDataset> rdf:type ja:MemoryDataset . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2a.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2a.ttl new file mode 100644 index 00000000000..6ad26b9835f --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2a.ttl @@ -0,0 +1,18 @@ +## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: + +PREFIX rdfs: +PREFIX ja: +PREFIX tdb2: + +<#service1> rdf:type fuseki:Service ; + fuseki:name "test-tdb2a" ; + fuseki:endpoint [ fuseki:name "sparql" ; + fuseki:operation fuseki:query ] ; + fuseki:dataset <#dataset> . + +<#dataset> rdf:type tdb2:DatasetTDB2 ; + tdb2:location "target/tdb2a" . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2b.ttl b/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2b.ttl new file mode 100644 index 00000000000..572927c331d --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Config/config-tdb2b.ttl @@ -0,0 +1,18 @@ +## Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +PREFIX : <#> +PREFIX fuseki: +PREFIX rdf: + +PREFIX rdfs: +PREFIX ja: +PREFIX tdb2: + +<#service1> rdf:type fuseki:Service ; + fuseki:name "test-tdb2b" ; + fuseki:endpoint [ fuseki:name "sparql" ; + fuseki:operation fuseki:query ] ; + fuseki:dataset <#dataset> . + +<#dataset> rdf:type tdb2:DatasetTDB2 ; + tdb2:location "target/tdb2b" . diff --git a/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_localhost.ini b/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_localhost.ini new file mode 100644 index 00000000000..2b9cce0c489 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_localhost.ini @@ -0,0 +1,13 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +[main] +localhostFilter = org.apache.jena.fuseki.authz.LocalhostFilter + +[users] +admin=pw + +[roles] + +[urls] +/local/** = localhostFilter +/public/** = anon diff --git a/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_userpassword.ini b/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_userpassword.ini new file mode 100644 index 00000000000..c0b57297263 --- /dev/null +++ b/jena-fuseki2/jena-fuseki-main/testing/Shiro/shiro_userpassword.ini @@ -0,0 +1,18 @@ +# Licensed under the terms of http://www.apache.org/licenses/LICENSE-2.0 + +[main] +plainMatcher=org.apache.shiro.authc.credential.SimpleCredentialsMatcher + +[users] +admin=pw + +[roles] + +[users] +admin=pw +user1=passwd1 + +[urls] +/$/ping = authcBasic,user[admin] +/ds = authcBasic,user[user1] +/**=anon