From e59a926f3f587c0e20b9f62726919a99a9350674 Mon Sep 17 00:00:00 2001 From: RemiFELIN Date: Tue, 6 Feb 2024 17:21:20 +0100 Subject: [PATCH] Probabilistic SHACL implementation --- .../fr/inria/corese/core/extension/Core.java | 2 + .../corese/core/extension/Extension.java | 14 +- .../fr/inria/corese/core/shacl/Shacl.java | 35 +++++ .../main/resources/function/datashape/core.rq | 50 +++++-- .../main/resources/function/datashape/main.rq | 92 ++++++++++++- .../resources/function/datashape/report.rq | 125 +++++++++++------- .../server/webservice/GraphProtocol.java | 4 +- .../server/webservice/SPARQLRestAPI.java | 67 ++++++---- .../server/webservice/SPARQLResult.java | 11 +- .../corese/server/webservice/TripleStore.java | 76 ++++++++++- .../sparql/triple/parser/NSManager.java | 2 + .../corese/sparql/triple/parser/URLParam.java | 5 + 12 files changed, 386 insertions(+), 97 deletions(-) diff --git a/corese-core/src/main/java/fr/inria/corese/core/extension/Core.java b/corese-core/src/main/java/fr/inria/corese/core/extension/Core.java index 3e522f52b1..5403eb4a90 100644 --- a/corese-core/src/main/java/fr/inria/corese/core/extension/Core.java +++ b/corese-core/src/main/java/fr/inria/corese/core/extension/Core.java @@ -70,6 +70,8 @@ static void define(String pref, String ns) { static void defNamespace() { prefix = new HashMap<>(); define("sh", NSManager.SHAPE); + // Probabilistic SHACL prefix + define("psh", NSManager.PROBSHACL); define("msh", MSH); } diff --git a/corese-core/src/main/java/fr/inria/corese/core/extension/Extension.java b/corese-core/src/main/java/fr/inria/corese/core/extension/Extension.java index 62ae6f55ab..0ff97e3e5d 100644 --- a/corese-core/src/main/java/fr/inria/corese/core/extension/Extension.java +++ b/corese-core/src/main/java/fr/inria/corese/core/extension/Extension.java @@ -337,6 +337,18 @@ int fib(int n) { } } - + /** + * @return the result of exp(dt), dt is a number + */ + public IDatatype exponential(IDatatype dt) { + return DatatypeMap.newInstance(Math.exp(dt.doubleValue())); + } + + /** + * @return PI value + */ + public IDatatype pi() { + return DatatypeMap.newInstance(Math.PI); + } } diff --git a/corese-core/src/main/java/fr/inria/corese/core/shacl/Shacl.java b/corese-core/src/main/java/fr/inria/corese/core/shacl/Shacl.java index 5c2cad4d49..d3b7c085b4 100644 --- a/corese-core/src/main/java/fr/inria/corese/core/shacl/Shacl.java +++ b/corese-core/src/main/java/fr/inria/corese/core/shacl/Shacl.java @@ -42,6 +42,12 @@ public class Shacl { static final String PARSE = SH + "funparse"; static final String SHACL = SH + "shacl"; + // Probabilistic SHACL + static final String EXTENDED_SHACL = SH + "extendedshacl"; + // modss + public static final int PROBABILISTIC_MODE = 1; + public static final int POSSIBILISTIC_MODE = 2; // not avalaible + static final String SHEX = SH + "shex"; static final String SHAPE = SH + "shaclshape"; static final String NODE = SH + "shaclnode"; @@ -209,6 +215,35 @@ public Graph eval(Graph shacl) throws EngineException { return eval(SHACL, shacl); } + /** + * probabilistic SHACL Evaluation (usage for Corese GUI) + * @return + * @throws EngineException + */ + public Graph eval(int mode) throws EngineException { + return eval(EXTENDED_SHACL, getShacl(), mode, DatatypeMap.createLiteral(String.valueOf(0.1), fr.inria.corese.sparql.datatype.RDF.xsddouble)); + } + + /** + * probabilistic SHACL Evaluation (usage for Corese server) with p-value + * @return + * @throws EngineException + */ + public Graph eval(Graph shacl, int mode, IDatatype p) throws EngineException { + setShacl(shacl); + return eval(EXTENDED_SHACL, shacl, mode, p); + } + + /** + * probabilistic SHACL Evaluation (usage for Corese server) with p-value and the number of considered triples + * @return + * @throws EngineException + */ + public Graph eval(Graph shacl, int mode, IDatatype p, IDatatype nTriples) throws EngineException { + setShacl(shacl); + return eval(EXTENDED_SHACL, shacl, mode, p, nTriples); + } + /** * Evaluate shape/node */ diff --git a/corese-core/src/main/resources/function/datashape/core.rq b/corese-core/src/main/resources/function/datashape/core.rq index b12dda9f43..18312b9823 100644 --- a/corese-core/src/main/resources/function/datashape/core.rq +++ b/corese-core/src/main/resources/function/datashape/core.rq @@ -35,6 +35,7 @@ function sh:core(report, sh, vis, nodeList, present) { sh:pop() } } ; + sh:log("core.rq", "sh:core", concat(sh, " is conform ? ", res)); return (res) } } @@ -323,18 +324,43 @@ function sh:coreboolean(shape, cst) { } } +# Probabilistic SHACL +# Binomial formulas +function sh:normDensity(x, mu, sigma) { + set(normDensity_exponent = -0.5 * power(((?x - ?mu) / ?sigma), 2)); + set(normDensity_coefficient = 1 / (?sigma * power(2 * fun:pi(), 0.5))); + set(normDensity_res = normDensity_coefficient * fun:exponential(normDensity_exponent)); + return (normDensity_res) +} +# Optimized combinaison (n k) +function sh:comb(n, k) { + # apply properties to optimize the computation + if(?n = ?k || ?k = 0) { + return (1) + }; + # not scalable : + # return (sh:fac(?n) / (sh:fac(?k) * sh:fac(?n - ?k))) + if(?k>?n-?k) { + ?k=?n - ?k; + }; + set(coef = 1); + for (i in xt:iota(0, ?k - 1)) { + set(coef=coef*(?n - i)); + set(coef=floor(coef/(i+1))); + }; + return (coef) +} +# P(X = k) +function sh:binomial(n, p, k) { + if(?n>=30 && ?n*?p>5 && ?n*(1-?p)>5) { + return (sh:normDensity(?k, ?n*?p, power(?n*?p*(1-?p), 0.5))) + }; + return (sh:comb(?n, ?k) * power(?p, ?k) * power(1 - ?p, ?n - ?k)) +} - - - - - - - - - - - - +# P(X >= k) +function sh:binomialEqualOrGreater(n, p, k) { + if(k = n, sh:binomial(?n, ?p, ?n), sh:binomial(?n, ?p, ?k) + sh:binomialEqualOrGreater(?n, ?p, ?k + 1)) +} diff --git a/corese-core/src/main/resources/function/datashape/main.rq b/corese-core/src/main/resources/function/datashape/main.rq index ce4402cd65..a6faf58965 100644 --- a/corese-core/src/main/resources/function/datashape/main.rq +++ b/corese-core/src/main/resources/function/datashape/main.rq @@ -6,7 +6,9 @@ prefix sh: prefix xsh: prefix shex: +prefix dct: prefix fun: +prefix psh: @import @@ -18,12 +20,14 @@ prefix fun: # function dt:graph sh:shacl() { if (sh:trace(), xt:print("shacl:"), true); + set(extended = false); sh:shacl(xt:graph(), sh:focus(xt:graph())) } # shape is shacl graph function dt:graph sh:shacl(dt:graph shape) { if (sh:trace(), xt:print("shacl:"), true); + set(extended = false); sh:shacl(shape, sh:focus(shape)) } @@ -37,6 +41,7 @@ function dt:graph sh:shaclshape(sh) { # Current graph is RDF and shacl graph function dt:graph sh:shaclShapeGraph(shacl, sh) { if (sh:trace(), xt:print("shaclshape:", sh), true); + set(extended = false); sh:shacl(shacl, sh:focus(shacl, sh)) } @@ -50,6 +55,7 @@ function dt:graph sh:shaclshape(sh, node) { function dt:graph sh:shaclShapeGraph(shacl, sh, node) { if (sh:trace(), xt:print("shaclshape:", sh, node), true); + set(extended = false); sh:shacl(shacl, xt:list(xt:list(sh, xt:list(node)))) } @@ -62,9 +68,52 @@ function dt:graph sh:shaclnode(node) { function dt:graph sh:shaclNodeGraph(shacl, node) { if (sh:trace(), xt:print("shaclnode:", node), true); + set(extended = false); sh:shacl(shacl, sh:focusnode(shacl, node)) } +# +# extended shacl function : probabilistic or possibilistic mode +# +function dt:graph sh:extendedshacl(dt:graph shape, m, p) { + if (sh:trace(), xt:print("shacl:"), true); + set(extended = true); + # define mod + set(mode = xsd:integer(m)); + # define p-value + set(prob = p); + # set default value and enable COUNT query on the RDF data graph + set(numInstances = 0); + # check the used mod + if (mode == 1, + sh:log("main.rq", "sh:extendedshacl", concat("Probabilistic mode used with p=", prob)), + if(mode == 2, + sh:log("main.rq", "sh:extendedshacl", "Possibilistic mode"), + sh:log("main.rq", "sh:extendedshacl", "ERROR this ID is not found !") + ) + ); + sh:shacl(shape, sh:focus(shape)); +} + +function dt:graph sh:extendedshacl(dt:graph shape, m, p, nTriples) { + if (sh:trace(), xt:print("shacl:"), true); + set(extended = true); + # define mod + set(mode = xsd:integer(m)); + # define p-value + set(prob = p); + # set the number of instances provided by user + set(numInstances = nTriples); + # check the used mod + if (mode == 1, + sh:log("main.rq", "sh:extendedshacl", concat("Probabilistic mode used with p= ", prob, " and n-triples= ", numInstances)), + if(mode == 2, + sh:log("main.rq", "sh:extendedshacl", "Possibilistic mode"), + sh:log("main.rq", "sh:extendedshacl", "ERROR this ID is not found !") + ) + ); + sh:shacl(shape, sh:focus(shape)); +} function dt:graph sh:shex() { @@ -115,20 +164,54 @@ function dt:graph sh:shacl(dt:graph shape, focus) { # focus = mappings or list(sh=nodeShape; list=targetNodeList) # function sh:shacleval(shape, focus) { + # compute the total number of instances in the graph if numInstances == 0 + if (extended && numInstances == 0) { + set(numInstances = sh:getNumInstances()); + } ; let (suc = true) { for ((sh list) in focus) { if (sh:trace(), xt:print("target:", coalesce(sh, "undef"), xt:size(list), list), true); + # set the final report as global variable + set(rep = sh:validationReport()); + set(abnode = sh:bnodeid()); if (bound(sh) && xt:size(list) > 0) { - let (res = sh:core(sh:validationReport(), sh, true, list)) { + sh:log("main.rq", "sh:shacleval", concat("xt:size(list)= ", xt:size(list))); + if(extended) { + # set default value for reference cardinality of current shape + set(referenceCardinality = 0); + # set the name of the given shape + set(shapeName = sh); + # For each shape, we will create a list of exceptions + set(exceptions = xt:list()); + } ; + # start core function + let (res = sh:core(rep, sh, true, list)) { + set(result = res); if (res, true, set(suc = false)) - } + } ; + # write the additionnal component (Extended SHACL) + if(extended) { + sh:addExtendedShaclReport(rep, exceptions); + } ; } + else { + sh:log("main.rq", "sh:shacleval", "The current shape is not considered: no nodes to be tested"); + } ; } ; return (suc) } } +# Get the total number of instances in the dataset +function sh:getNumInstances() { + let(select (count(*) as ?n) where { + ?s ?p ?o . + }) { + return (n) + } +} + function sh:focus() { sh:focuslist(xt:graph()) } @@ -731,4 +814,7 @@ function sh:checkinit() { if (bound(mapmap), true, sh:start(xt:graph())) } - +# Log a specific msg according to the parameters +function sh:log(file, m, msg) { + xt:print(now()," - [",file,"] - ",m," - ",msg); +} diff --git a/corese-core/src/main/resources/function/datashape/report.rq b/corese-core/src/main/resources/function/datashape/report.rq index 41271a73ff..bae262805c 100644 --- a/corese-core/src/main/resources/function/datashape/report.rq +++ b/corese-core/src/main/resources/function/datashape/report.rq @@ -3,6 +3,7 @@ # # Olivier Corby - Wimmics Inria I3S - 2016-2020 # +prefix psh: prefix sh: prefix xsh: prefix shex: @@ -25,6 +26,10 @@ function xsd:boolean sh:report(report, name, sh, cst, s, p, o, suc, vis){ function xsd:boolean sh:report(report, subReport, name, sh, cst, s, p, o, suc, vis){ sh:record(name, sh, cst, sh:shaclGraph(), s, p, o, suc, vis); + # increase reference cardinality (extended SHACL) + if(extended == true) { + set(referenceCardinality = referenceCardinality + 1); + } ; if (vis && ! suc) { sh:visit(name, s, suc) ; return (sh:result(report, subReport, name, sh, cst, sh:shaclGraph(), s, p, o)) ; @@ -46,60 +51,78 @@ function xsd:boolean sh:result(report, subReport, name, sh, cst, dt:graph shape, def = coalesce(sh:isdefby(name), name), sev = coalesce(xt:value(shape, sh, sh:severity), sh:Violation), mes = sh:getTheMessage(shape, sh, cst, foc, node), - mesfun = sh:messageFunction(shape, sh, cst, foc, node), - abnode = sh:bnodeid()) { - - sh:store(report, abnode, rdf:type, sh:ValidationReport); - sh:store(report, abnode, sh:conforms, false); - - if (report = sh:validationReport()) { + mesfun = sh:messageFunction(shape, sh, cst, foc, node)) { + + # abnode is defined in main.rq -> sh:shacleval(shape, focus) + sh:store(report, abnode, rdf:type, sh:ValidationReport); + sh:store(report, abnode, sh:conforms, false); + # set the global evaluation report as false + set(fail = true); + if (report = sh:validationReport()) { sh:store(report, abnode, sh:result, res); sh:store(report, res, rdf:type, sh:ValidationResult) - } - else { + } + else { sh:store(report, res, rdf:type, sh:AbstractResult) - } ; - - sh:store(report, res, sh:resultSeverity, sev); - sh:store(report, res, sh:focusNode, foc); - - sh:store(report, res, sh:resultMessage, mes) ; - if (coalesce(mesfun = "", false), true, - sh:store(report, res, sh:resultMessage2, mesfun)) ; - - sh:store(report, res, sh:sourceConstraintComponent, def) ; - sh:store(report, res, sh:sourceShape, sh) ; - - if (sh:isValue(path)) { + } ; + sh:store(report, res, sh:resultSeverity, sev); + sh:store(report, res, sh:focusNode, foc); + sh:store(report, res, sh:resultMessage, mes) ; + if (coalesce(mesfun = "", false), true, sh:store(report, res, sh:resultMessage2, mesfun)); + sh:store(report, res, sh:sourceConstraintComponent, def) ; + sh:store(report, res, sh:sourceShape, shapeName) ; + if (sh:isValue(path)) { sh:store(report, res, sh:resultPath, sh:prettyNodeOrList(shape, path)) - } ; - if (sh:isValue(node)) { + }; + if (sh:isValue(node)) { sh:store(report, res, sh:value, node) - } ; - - if (isBlank(node)) { + }; + if (isBlank(node)) { sh:store(report, res, sh:valueDetail, sh:graphdt(shape, node)) - } ; - if (isBlank(foc) && ! sh:isFast()) { + }; + if (isBlank(foc) && ! sh:isFast()) { sh:store(report, res, sh:focusNodeDetail, sh:graphdt(shape, foc)) - }; - if (isBlank(sh)) { - #sh:document(sh); + }; + if (isBlank(sh)) { coalesce(sh:store(report, res, sh:sourceShapeDetail, sh:getDefinition(sh)), true) - }; - - sh:resultFunction(report, subReport, res, sh, name, foc, path, node); - - return (res) - + }; + # Extended SHACL + if(extended == true && sev == sh:Violation) { + # fill the list of exceptions + xt:add(exceptions, res); + }; + sh:resultFunction(report, subReport, res, sh, name, foc, path, node); + return (res) } ; - #return (true) } - - - - +# Extended SHACL Report component +function sh:addExtendedShaclReport(report, exceptions) { + # sh:log("core.rq", "sh:binomial", "extended report !"); + set(summary = bnode()); + sh:store(report, abnode, psh:summary, summary); + sh:store(report, summary, rdf:type, psh:ValidationSummary); + # sh:store(report, summary, psh:mode, "Probabilistic"); + # shapeName is defined on sh:shacleval(shape, focus) + sh:store(report, summary, psh:focusShape, shapeName); + # referenceCardinality is defined on sh:shacleval(shape, focus) + sh:store(report, summary, psh:referenceCardinality, referenceCardinality); + # numException is defined by the size of exceptions list + set(numException = xt:size(exceptions)); + sh:store(report, summary, psh:numViolation, numException); + # numConfirmation is defined by referenceCardinality and the size of exceptions list + set(numConfirmation = (referenceCardinality - numException)); + # fix negative values for the number of confirmations + if(numConfirmation < 0) { + set(numConfirmation = 0); + }; + sh:store(report, summary, psh:numConfirmation, numConfirmation); + # sh:log("core.rq", "sh:binomial", "start binomiale !"); + set(likelihood = sh:binomial(referenceCardinality, prob, numException)); + sh:store(report, summary, psh:likelihood, likelihood); + set(generality = (referenceCardinality / numInstances)); + sh:store(report, summary, psh:generality, generality); +} # # Additional report @@ -143,12 +166,18 @@ function sh:tracerecord(shape, mapmap) { # finish function xsd:boolean sh:success(xsd:boolean bb) { if (bb) { - let (g = - construct { [] a sh:ValidationReport ; sh:conforms true } - where { } - ) { - sh:insert(g, sh:validationReport()) + if (extended) { + sh:store(rep, abnode, rdf:type, sh:ValidationReport); + sh:store(rep, abnode, sh:conforms, true); } + else { + let (g = + construct { [] a sh:ValidationReport ; sh:conforms true } + where { } + ) { + sh:insert(g, sh:validationReport()) + } + } ; } ; return (true) } diff --git a/corese-server/src/main/java/fr/inria/corese/server/webservice/GraphProtocol.java b/corese-server/src/main/java/fr/inria/corese/server/webservice/GraphProtocol.java index f28b6029bd..9078c58902 100644 --- a/corese-server/src/main/java/fr/inria/corese/server/webservice/GraphProtocol.java +++ b/corese-server/src/main/java/fr/inria/corese/server/webservice/GraphProtocol.java @@ -41,7 +41,7 @@ Response get(HttpServletRequest request, String name, String graph, String patte query = String.format(pattern, NSManager.nsm().toNamespace(graph)); } return new SPARQLRestAPI().myGetResult(request, name, null, null, null, null, query, access, null, null, - format); + format, null); } Response post(HttpServletRequest request, String name, String graph, String pattern, String access, int format) { @@ -52,7 +52,7 @@ Response post(HttpServletRequest request, String name, String graph, String patt query = String.format(NAMED_GRAPH_INSERT, NSManager.nsm().toNamespace(graph), pattern); } return new SPARQLRestAPI().myGetResult(request, name, null, null, null, null, query, access, null, null, - format); + format, null); } String getQuery(String name) { diff --git a/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLRestAPI.java b/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLRestAPI.java index c826c5ec04..9db83d4e57 100644 --- a/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLRestAPI.java +++ b/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLRestAPI.java @@ -3,6 +3,7 @@ import java.util.List; import java.util.UUID; +import org.apache.jena.sparql.pfunction.library.concat; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -319,16 +320,16 @@ public Response getTriplesXMLForGet(@jakarta.ws.rs.core.Context HttpServletReque // if there is header accept, the value of format is overloaded by header accept // if there is no header and no format, default format is chosen (xml or turtle) return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, format, - UNDEF_FORMAT, transform); + UNDEF_FORMAT, transform, null); } public Response getResultFormat(HttpServletRequest request, String name, String oper, List uri, List param, List mode, String query, String access, List defaut, List named, - String format, int type, List transform) { + String format, int type, List transform, String content) { return new SPARQLResult(request).setVisitor(getVisitor()) - .getResultFormat(name, oper, uri, param, mode, query, access, defaut, named, format, type, transform); + .getResultFormat(name, oper, uri, param, mode, query, access, defaut, named, format, type, transform, content); } /** @@ -341,8 +342,8 @@ Response myGetResult(HttpServletRequest request, String name, String oper, List uri, List param, List mode, String query, String access, List defaut, List named, - int type) { - return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, type, null); + int type, String content) { + return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, type, null, content); } Response getResultForPost(HttpServletRequest request, @@ -354,8 +355,9 @@ Response getResultForPost(HttpServletRequest request, String query, String access, List defaut, List named, - int format) { - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, format); + int format, + String content) { + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, format, content); } /** @@ -400,7 +402,7 @@ public Response getHTMLForGet(@jakarta.ws.rs.core.Context HttpServletRequest req null, null, null, null, format, access, query, null, null, null, defaut, named); } return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, HTML_FORMAT, - transform); + transform, null); } @GET @@ -417,7 +419,7 @@ public Response getTriplesXMLForGet2(@jakarta.ws.rs.core.Context HttpServletRequ @QueryParam("uri") List uri) { logger.info("getTriplesXMLForGet2"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT, null); } @GET @@ -436,7 +438,7 @@ public Response getTriplesJSONForGet(@jakarta.ws.rs.core.Context HttpServletRequ logger.info("getTriplesJSONForGet"); return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, JSON_FORMAT, - transform); + transform, null); } @GET @@ -453,7 +455,7 @@ public Response getTriplesCSVForGet(@jakarta.ws.rs.core.Context HttpServletReque @QueryParam("uri") List uri) { logger.info("getTriplesCSVForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, CSV_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, CSV_FORMAT, null); } @GET @@ -470,7 +472,7 @@ public Response getTriplesTSVForGet(@jakarta.ws.rs.core.Context HttpServletReque @QueryParam("uri") List uri) { logger.info("getTriplesTSVForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TSV_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TSV_FORMAT, null); } // ---------------------------------------------------- @@ -491,7 +493,7 @@ public Response getRDFGraphXMLForGet(@jakarta.ws.rs.core.Context HttpServletRequ @QueryParam("uri") List uri) { logger.info("getRDFGraphXMLForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, RDF_XML_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, RDF_XML_FORMAT, null); } @GET @@ -508,7 +510,7 @@ public Response getRDFGraphNTripleForGet(@jakarta.ws.rs.core.Context HttpServlet @QueryParam("uri") List uri) { logger.info("getRDFGraphNTripleForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TURTLE_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TURTLE_FORMAT, null); } @GET @@ -525,7 +527,7 @@ public Response getRDFGraphTrigForGet(@jakarta.ws.rs.core.Context HttpServletReq @QueryParam("uri") List uri) { logger.info("getRDFGraphTrigForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TRIG_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, TRIG_FORMAT, null); } @GET @@ -542,7 +544,7 @@ public Response getRDFGraphJsonLDForGet(@jakarta.ws.rs.core.Context HttpServletR @QueryParam("uri") List uri) { logger.info("getRDFGraphJsonLDForGet"); - return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, JSONLD_FORMAT); + return myGetResult(request, name, oper, uri, param, mode, query, access, defaut, named, JSONLD_FORMAT, null); } // ---------------------------------------------------- @@ -570,13 +572,14 @@ public Response getXMLForPost(@jakarta.ws.rs.core.Context HttpServletRequest req String message, @QueryParam("param") List param, @QueryParam("mode") List mode, + @FormParam("content") String content, @QueryParam("uri") List uri) { logger.info("getXMLForPost"); query = getQuery(query, message); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, XML_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, XML_FORMAT, content); } @POST @@ -592,13 +595,14 @@ public Response getXMLForPostText(@jakarta.ws.rs.core.Context HttpServletRequest String message, @QueryParam("param") List param, @QueryParam("mode") List mode, + @FormParam("content") String content, @QueryParam("uri") List uri) { logger.info("getXMLForPostText"); query = getQuery(query, message); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT, content); } /** @@ -624,6 +628,7 @@ public Response getTriplesXMLForPost(@jakarta.ws.rs.core.Context HttpServletRequ @FormParam("transform") List transform, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { @@ -641,7 +646,7 @@ public Response getTriplesXMLForPost(@jakarta.ws.rs.core.Context HttpServletRequ // dataset(defaut, using), dataset(named, usingNamed) return getResultFormat(request, name, oper, uri, param, mode, query, - access, defaut, named, format, UNDEF_FORMAT, transform); + access, defaut, named, format, UNDEF_FORMAT, transform, content); } List dataset(List from, List using) { @@ -664,12 +669,13 @@ public Response getTriplesTEXTForPost(@jakarta.ws.rs.core.Context HttpServletReq @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getTriplesTEXTForPost"); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TEXT_FORMAT, content); } @POST @@ -684,12 +690,13 @@ public Response getTriplesJSONForPostNew(@jakarta.ws.rs.core.Context HttpServlet @QueryParam("named-graph-uri") List named, @QueryParam("param") List param, @QueryParam("mode") List mode, + @FormParam("content") String content, @QueryParam("uri") List uri, String message) { logger.info("getTriplesJSONForPostNew"); query = getQuery(query, message); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, JSON_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, JSON_FORMAT, content); } @POST @@ -706,13 +713,14 @@ public Response getTriplesJSONForPost(@jakarta.ws.rs.core.Context HttpServletReq @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getTriplesJSONForPost"); return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, JSON_FORMAT, - transform); + transform, content); } @POST @@ -729,13 +737,14 @@ public Response getTriplesCSVForPost(@jakarta.ws.rs.core.Context HttpServletRequ @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getTriplesCSVForPost"); return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, CSV_FORMAT, - transform); + transform, content); } // try { @@ -767,12 +776,13 @@ public Response getTriplesTSVForPost(@jakarta.ws.rs.core.Context HttpServletRequ @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getTriplesTSVForPost"); return getResultFormat(request, name, oper, uri, param, mode, query, access, defaut, named, null, TSV_FORMAT, - transform); + transform, content); } // try { @@ -807,13 +817,14 @@ public Response getRDFGraphXMLForPost(@jakarta.ws.rs.core.Context HttpServletReq @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getRDFGraphXMLForPost"); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, RDF_XML_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, RDF_XML_FORMAT, content); } @POST @@ -829,12 +840,13 @@ public Response getRDFGraphNTripleForPost(@jakarta.ws.rs.core.Context HttpServle @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getRDFGraphNTripleForPost"); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TURTLE_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, TURTLE_FORMAT, content); } @POST @@ -850,11 +862,12 @@ public Response getRDFGraphJsonLDForPost(@jakarta.ws.rs.core.Context HttpServlet @FormParam("named-graph-uri") List named, @FormParam("param") List param, @FormParam("mode") List mode, + @FormParam("content") String content, @FormParam("uri") List uri, String message) { query = getQuery(query, update, message); logger.info("getRDFGraphJsonLDForPost"); - return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, JSONLD_FORMAT); + return getResultForPost(request, name, oper, uri, param, mode, query, access, defaut, named, JSONLD_FORMAT, content); } diff --git a/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLResult.java b/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLResult.java index 7f83ac7904..5c1ce89d48 100644 --- a/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLResult.java +++ b/corese-server/src/main/java/fr/inria/corese/server/webservice/SPARQLResult.java @@ -84,7 +84,7 @@ public Response getResultFormat(String name, String oper, List uri, List param, List mode, String query, String access, List defaut, List named, - String format, int type, List transform) { + String format, int type, List transform, String content) { try { logger.info("Endpoint URL: " + getRequest().getRequestURL()); @@ -98,7 +98,7 @@ public Response getResultFormat(String name, String oper, beforeRequest(getRequest(), query); Dataset ds = createDataset(getRequest(), defaut, named, access); - beforeParameter(ds, oper, uri, param, mode, transform); + beforeParameter(ds, oper, uri, param, mode, transform, content); Mappings map = getTripleStore(name).query(getRequest(), query, ds); complete(map, ds.getContext()); afterParameter(ds, map); @@ -189,7 +189,7 @@ Dataset createDataset(HttpServletRequest request, List defaut, List uri, - List param, List mode, List transform) { + List param, List mode, List transform, String content) { if (oper != null) { ds.getContext().set(OPER, oper); List federation = new ArrayList<>(); @@ -283,6 +283,11 @@ Dataset beforeParameter(Dataset ds, String oper, List uri, } } + // to use prob-shacl service directly with shapes as string + if(content != null) { + ds.getContext().set(URLParam.CONTENT, decode(content)); + } + if (mode != null) { for (String kw : mode) { // decode mode=map diff --git a/corese-server/src/main/java/fr/inria/corese/server/webservice/TripleStore.java b/corese-server/src/main/java/fr/inria/corese/server/webservice/TripleStore.java index 2b47cea032..a0c6012068 100644 --- a/corese-server/src/main/java/fr/inria/corese/server/webservice/TripleStore.java +++ b/corese-server/src/main/java/fr/inria/corese/server/webservice/TripleStore.java @@ -1,5 +1,9 @@ package fr.inria.corese.server.webservice; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Date; import java.util.Enumeration; import java.util.HashMap; @@ -27,6 +31,7 @@ import fr.inria.corese.sparql.triple.parser.Dataset; import fr.inria.corese.sparql.triple.parser.Metadata; import fr.inria.corese.sparql.triple.parser.URLParam; +import fr.inria.corese.sparql.datatype.RDF; import jakarta.servlet.http.Cookie; import jakarta.servlet.http.HttpServletRequest; @@ -192,7 +197,7 @@ Mappings query(HttpServletRequest request, String query, Dataset ds) throws Engi QueryProcess exec = getQueryProcess(); exec.setDebug(c.isDebug()); - Mappings map; + Mappings map = null; try { before(exec, query, ds); TripleStoreLog tsl = new TripleStoreLog(exec, c); @@ -211,6 +216,8 @@ Mappings query(HttpServletRequest request, String query, Dataset ds) throws Engi } } else if (isShacl(c)) { map = shacl(query, ds); + } else if (isProbabilisticShacl(c)) { + map = probabilisticShacl(query, ds); } else if (isConstruct(c)) { map = construct(query, ds); } else if (isSpin(c)) { @@ -227,6 +234,8 @@ Mappings query(HttpServletRequest request, String query, Dataset ds) throws Engi } else { throw e; } + } catch (IOException e) { + e.printStackTrace(); } // add param=value parameter to Context @@ -319,6 +328,11 @@ boolean isShacl(Context c) { return c.hasValue(SHACL) && hasValueList(c, URI); } + boolean isProbabilisticShacl(Context c) { + if(c.hasValue(PROB_SHACL) && c.hasValue(CONTENT)) return true; + else return c.hasValue(PROB_SHACL) && hasValueList(c, URI); + } + boolean isConstruct(Context c) { return c.hasValue(CONSTRUCT); } @@ -369,6 +383,66 @@ Mappings shacl(String query, Dataset ds) throws EngineException { return map; } + /** + * sparql?mode=prob-shacl¶m=p:[value],n-triples:[value]&query=select * where { ?s sh:conforms ?b } + * sparql?mode=prob-shacl&uri=uri&query=select * where { ?s sh:conforms ?b } + * Load shacl shape + * Evaluate shape + * Execute query on shacl validation report + */ + Mappings probabilisticShacl(String query, Dataset ds) throws EngineException, IOException { + Graph shacl = Graph.create(); + Load ld = Load.create(shacl); + double p = 0; + Integer nTriples = null; + try { + if(ds.getContext().get(URLParam.CONTENT) != null) { + InputStream stream = new ByteArrayInputStream(ds.getContext().get(URLParam.CONTENT).stringValue().getBytes(StandardCharsets.UTF_8)); + ld.parse(stream, "", Load.TURTLE_FORMAT); + stream.close(); + } else { + for (IDatatype dt : ds.getContext().get(URLParam.URI)) { + ld.parse(dt.getLabel()); + } + } + // Manage params + if(ds.getContext().get(PARAM) != null) { + // parsing params as: p=[value];n-triples=[value] + for (IDatatype paramDt : ds.getContext().get(PARAM).getValueList()) { + String param = paramDt.getLabel(); + // p-value + if(param.contains(PROB_SHACL_P)) { + p = Double.parseDouble(param.replace(PROB_SHACL_P + ":", "")); + } + // n-triples + if(param.contains(PROB_SHACL_N_TRIPLES)) { + nTriples = Integer.parseInt(param.replace(PROB_SHACL_N_TRIPLES + ":", "")); + } + } + } + + } catch (LoadException ex) { + logger.error(ex.getMessage()); + throw new EngineException(ex) ; + } + logger.info("The prob-shacl validator will use the default p-value: p=" + p); + Shacl sh = new Shacl(getGraph()); + sh.setDataManager(getDataManager()); + Graph res; + if (nTriples != null) { + logger.info("The prob-shacl validator will consider the total number of RDF triples: n-triples=" + nTriples); + res = sh.eval(shacl, Shacl.PROBABILISTIC_MODE, + DatatypeMap.createLiteral(String.valueOf(p), RDF.xsddouble), + DatatypeMap.createLiteral(String.valueOf(nTriples), RDF.xsdinteger)); + } else { + res = sh.eval(shacl, Shacl.PROBABILISTIC_MODE, DatatypeMap.createLiteral(String.valueOf(p), RDF.xsddouble)); + } + QueryProcess exec = QueryProcess.create(res); + exec.setDebug(ds.getContext().isDebug()); + Mappings map = exec.query(query); + return map; + } + Mappings construct(String query, Dataset ds) throws EngineException { Graph g = Graph.create(); Load ld = Load.create(g); diff --git a/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/NSManager.java b/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/NSManager.java index 4669cc651e..d458421897 100755 --- a/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/NSManager.java +++ b/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/NSManager.java @@ -93,6 +93,8 @@ public class NSManager extends ASTObject { public static final String SHAPE = SHACL; public static final String SHACL_JAVA = "function://fr.inria.corese.core.extension.SHACL."; public static final String SHACL_SHACL = RESOURCE + "data/shaclshacl.ttl"; + // Probabilistic SHACL + public static final String PROBSHACL = "http://ns.inria.fr/probabilistic-shacl/"; public static final String COSNS = RDFS.COSNS; public static final String COS = RDFS.COS; diff --git a/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/URLParam.java b/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/URLParam.java index 5ce76f8247..b8cd481a85 100644 --- a/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/URLParam.java +++ b/sparql/src/main/java/fr/inria/corese/sparql/triple/parser/URLParam.java @@ -22,6 +22,11 @@ public interface URLParam { static final String URL = "url"; static final String DEFAULT_GRAPH = "default-graph-uri"; static final String NAMED_GRAPH = "named-graph-uri"; + // Probabilistic SHACL + static final String PROB_SHACL = "prob-shacl"; + static final String PROB_SHACL_P = "p"; + static final String PROB_SHACL_N_TRIPLES = "nT"; + static final String CONTENT = "content"; // specific for service clause static final String LOOP = "loop";