Skip to content

Commit

Permalink
Merge pull request #20 from NASA-PDS/pds-api-125
Browse files Browse the repository at this point in the history 
pds-api-125: get the XML MIME types working again
  • Loading branch information
@tloubrieu-jpl
tloubrieu-jpl authored Jan 26, 2022
2 parents a72b40e + c3cd3e0 commit 57f4aa1
Show file tree
Hide file tree
Showing 28 changed files with 236 additions and 782 deletions.
8 changes: 4 additions & 4 deletions lexer/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ POSSIBILITY OF SUCH DAMAGE.
<modelVersion>4.0.0</modelVersion>

<groupId>gov.nasa.pds</groupId>
<artifactId>api-search-query-lexer</artifactId>
<artifactId>registry-api-lexer</artifactId>
<version>1.1.0-SNAPSHOT</version>
<packaging>jar</packaging>

Expand Down Expand Up @@ -170,9 +170,9 @@ POSSIBILITY OF SUCH DAMAGE.
</build>

<scm>
<url>https://github.com/NASA-PDS/api-search-query-lexer/tree/master</url>
<connection>scm:git:git://github.com/NASA-PDS/api-search-query-lexer.git</connection>
<developerConnection>scm:git:[email protected]:NASA-PDS/api-search-query-lexer.git</developerConnection>
<url>https://github.com/NASA-PDS/registry-api/tree/main/lexer</url>
<connection>scm:git:git://github.com/NASA-PDS/registry-api.git</connection>
<developerConnection>scm:git:[email protected]:NASA-PDS/registry-api.git</developerConnection>
<tag>v1.0.0</tag>
</scm>

Expand Down
23 changes: 11 additions & 12 deletions service/docker/Dockerfile.local
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,25 +14,24 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
tar

# Make room for the app
RUN mkdir /usr/local/registry-api-service-${VERSION}
RUN mkdir -p /usr/local/registry-${VERSION}

# Copy the data into the building container
COPY LICENSE.md /usr/local/registry-api-service-${VERSION}
COPY NOTICE.txt /usr/local/registry-api-service-${VERSION}
COPY README.md /usr/local/registry-api-service-${VERSION}
COPY pom.xml /usr/local/registry-api-service-${VERSION}
#COPY m2.tgz /tmp
#RUN cd /root && tar -zxf /tmp/m2.tgz
COPY src /usr/local/registry-api-service-${VERSION}/src
COPY LICENSE.md /usr/local/registry-${VERSION}/
COPY pom.xml /usr/local/registry-${VERSION}/
COPY SECURITY.md /usr/local/registry-${VERSION}/
COPY lexer /usr/local/registry-${VERSION}/lexer
COPY model /usr/local/registry-${VERSION}/model
COPY service /usr/local/registry-${VERSION}/service

# Resources shared with the rest of the world
EXPOSE 8080

# Build the application and deploy it inside the container
RUN cd /usr/local/registry-api-service-${VERSION} && \
mvn clean && \
mvn install
RUN set -x && \
cd /usr/local/registry-${VERSION} && \
mvn clean install

# Run the sevice by default
WORKDIR /usr/local/registry-api-service-${VERSION}
WORKDIR /usr/local/registry-${VERSION}/service
CMD ["mvn", "spring-boot:run"]
14 changes: 3 additions & 11 deletions service/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,14 +250,6 @@ POSSIBILITY OF SUCH DAMAGE.
<version>3.25.0-GA</version>
</dependency>

<!-- used to force a specific namespace in the XML serialization -->
<dependency>
<groupId>org.codehaus.woodstox</groupId>
<artifactId>woodstox-core-asl</artifactId>
<version>4.4.0</version>
</dependency>


<!-- https://mvnrepository.com/artifact/org.threeten/threetenbp -->
<dependency>
<groupId>org.threeten</groupId>
Expand All @@ -267,13 +259,13 @@ POSSIBILITY OF SUCH DAMAGE.

<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>api</artifactId>
<version>0.5.0-SNAPSHOT</version>
<artifactId>registry-api-model</artifactId>
<version>0.5.0-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>gov.nasa.pds</groupId>
<artifactId>api-search-query-lexer</artifactId>
<artifactId>registry-api-lexer</artifactId>
<version>1.1.0-SNAPSHOT</version>
</dependency>

Expand Down
11 changes: 6 additions & 5 deletions service/src/main/java/gov/nasa/pds/api/engineering/configuration/WebMVCConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@
import gov.nasa.pds.api.engineering.serializer.Pds4JsonProductSerializer;
import gov.nasa.pds.api.engineering.serializer.Pds4JsonProductsSerializer;
import gov.nasa.pds.api.engineering.serializer.Pds4XmlProductSerializer;
import gov.nasa.pds.api.engineering.serializer.Pds4XmlProductsSerializer;
import gov.nasa.pds.api.engineering.serializer.PdsProductTextHtmlSerializer;
import gov.nasa.pds.api.engineering.serializer.PdsProductXMLSerializer;
import gov.nasa.pds.api.engineering.serializer.PdsProductsTextHtmlSerializer;

import gov.nasa.pds.api.engineering.serializer.XmlProductSerializer;

import gov.nasa.pds.api.engineering.serializer.PdsProductsXMLSerializer;

@Configuration
@EnableWebMvc
Expand Down Expand Up @@ -73,9 +73,10 @@ public void configureMessageConverters(List<HttpMessageConverter<?>> converters)
converters.add(new Pds4JsonProductSerializer());
converters.add(new Pds4JsonProductsSerializer());
converters.add(new Pds4XmlProductSerializer());
converters.add(new Pds4XmlProductsSerializer());
converters.add(new PdsProductTextHtmlSerializer());
converters.add(new PdsProductXMLSerializer());
converters.add(new PdsProductsTextHtmlSerializer());
converters.add(new XmlProductSerializer()); // Product class, application/xml
//converters.add(new Jaxb2RootElementHttpMessageConverter()); // other classes, application/xml
converters.add(new PdsProductsXMLSerializer());
}
}
11 changes: 3 additions & 8 deletions ...e/src/main/java/gov/nasa/pds/api/engineering/controllers/MyProductsApiBareController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@
import javax.servlet.http.HttpServletRequest;

import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -171,11 +168,9 @@ protected ResponseEntity<Object> getLatestProductResponseEntity(String lidvid)
{
lidvid = this.productBO.getLidVidDao().getLatestLidVidByLid(lidvid);
RequestAndResponseContext context = RequestAndResponseContext.buildRequestAndResponseContext(this.objectMapper, this.getBaseURL(), lidvid, this.presetCriteria, accept);
GetRequest request = new GetRequest(this.esRegistryConnection.getRegistryIndex(), lidvid);
FetchSourceContext fetchSourceContext = new FetchSourceContext(true, context.getFields().toArray(new String[0]), null);
request.fetchSourceContext(fetchSourceContext);
context.setResponse(this.esRegistryConnection.getRestHighLevelClient().get(request, RequestOptions.DEFAULT));

SearchRequest request = ElasticSearchRegistrySearchRequestBuilder.getQueryFieldsFromKVP("lidvid", lidvid, context.getFields(),this.esRegistryConnection.getRegistryIndex(), false);
context.setResponse(this.esRegistryConnection.getRestHighLevelClient(), request);

if (context.getResponse() == null)
{
log.warn("Could not find any matches for LIDVID: " + lidvid);
Expand Down
118 changes: 35 additions & 83 deletions ...gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchRegistrySearchRequestBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,135 +5,84 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.lang.Math;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.PrefixQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;

import gov.nasa.pds.api.engineering.elasticsearch.business.CollectionProductRefBusinessObject;
import gov.nasa.pds.api.engineering.elasticsearch.business.ProductQueryBuilderUtil;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob;


public class ElasticSearchRegistrySearchRequestBuilder {
public class ElasticSearchRegistrySearchRequestBuilder
{
private static final Logger log = LoggerFactory.getLogger(ElasticSearchRegistrySearchRequestBuilder.class);
private static final String[] DEFAULT_ALL_FIELDS = { "*" };

private static final String[] DEFAULT_BLOB = { "ops:Label_File_Info/ops:blob" };

private String registryIndex;
private String registryRefIndex;
final public String registryIndex;
final public String registryRefIndex;
private int timeOutSeconds;

public ElasticSearchRegistrySearchRequestBuilder(
String registryIndex,
String registryRefindex,
int timeOutSeconds) {
int timeOutSeconds)
{
this.registryIndex = registryIndex;
this.registryRefIndex = registryRefindex;
this.timeOutSeconds = timeOutSeconds;

}



public ElasticSearchRegistrySearchRequestBuilder() {

public ElasticSearchRegistrySearchRequestBuilder()
{
this.registryIndex = "registry";
this.registryRefIndex = "registry-refs";
this.timeOutSeconds = 60;

}

static private String[] excludes (List<String> fields)
{
String[] exclude, ex0 = new String[0], exbp = {EntitytProductWithBlob.BLOB_PROPERTY},
exjbp = {EntitytProductWithBlob.JSON_BLOB_PROPERTY},
exall = {EntitytProductWithBlob.BLOB_PROPERTY, EntitytProductWithBlob.JSON_BLOB_PROPERTY};

public SearchRequest getSearchProductRefsFromCollectionLidVid(
String lidvid,
int start,
int limit) {
MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("collection_lidvid", lidvid);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();

int productRefStart = (int)Math.floor(start/(float)CollectionProductRefBusinessObject.PRODUCT_REFERENCES_BATCH_SIZE);
int productRefLimit = (int)Math.ceil(limit/(float)CollectionProductRefBusinessObject.PRODUCT_REFERENCES_BATCH_SIZE);
log.debug("Request product reference documents from " + Integer.toString(productRefStart) + " for size " + Integer.toString(productRefLimit) );
searchSourceBuilder.query(matchQueryBuilder)
.from(productRefStart)
.size(productRefLimit);
SearchRequest searchRequest = new SearchRequest();
searchRequest.source(searchSourceBuilder);
searchRequest.indices(this.registryRefIndex);
if (fields.contains(EntitytProductWithBlob.BLOB_PROPERTY) && fields.contains(EntitytProductWithBlob.JSON_BLOB_PROPERTY)) exclude = ex0;
else if (fields.contains(EntitytProductWithBlob.BLOB_PROPERTY)) exclude = exjbp;
else if (fields.contains(EntitytProductWithBlob.JSON_BLOB_PROPERTY)) exclude = exbp;
else exclude = exall;


log.debug("search product ref request :" + searchRequest.toString());

return searchRequest;

return exclude;
}

public SearchRequest getSearchProductRequestHasLidVidPrefix(String lidvid) {

public SearchRequest getSearchProductRequestHasLidVidPrefix(String lidvid)
{
PrefixQueryBuilder prefixQueryBuilder = QueryBuilders.prefixQuery("lidvid", lidvid);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(prefixQueryBuilder);
SearchRequest searchRequest = new SearchRequest();
searchRequest.source(searchSourceBuilder);
searchRequest.indices(this.registryIndex);

return searchRequest;
}


public SearchRequest getSearchProductsByLid(String lid, int from, int size)
{
TermQueryBuilder termQuery = QueryBuilders.termQuery("lid", lid);
SearchSourceBuilder srcBuilder = new SearchSourceBuilder();
srcBuilder.query(termQuery);
srcBuilder.from(from);
srcBuilder.size(size);

SearchRequest searchRequest = new SearchRequest();
searchRequest.source(srcBuilder);
searchRequest.indices(this.registryIndex);

return searchRequest;
}



public GetRequest getGetProductRequest(String lidvid, boolean withXMLBlob) {

GetRequest getProductRequest = new GetRequest(this.registryIndex,
lidvid);

FetchSourceContext fetchSourceContext = new FetchSourceContext(true, null, withXMLBlob?null:new String[] { EntitytProductWithBlob.BLOB_PROPERTY });

getProductRequest.fetchSourceContext(fetchSourceContext);

return getProductRequest;

}


public GetRequest getGetProductRequest(String lidvid) {

return this.getGetProductRequest(lidvid, false);

}



public SearchRequest getSearchProductsRequest(
String queryString,
Expand All @@ -155,8 +104,8 @@ public SearchRequest getSearchProductsRequest(
query = ProductQueryBuilderUtil.createPqlQuery(queryString, fields, presetCriteria);
}

String[] excludedFields = excludes (fields);
String[] includedFields = fields.toArray(new String[0]);
String[] excludedFields = { EntitytProductWithBlob.BLOB_PROPERTY };

SearchRequestBuilder bld = new SearchRequestBuilder(query, start, limit);
bld.fetchSource(true, includedFields, excludedFields);
Expand All @@ -179,7 +128,7 @@ public SearchRequest getSearchCollectionRequest(String queryString, String keywo
return getSearchProductsRequest(queryString, keyword, fields, start, limit, presetCriteria);
}

static public SearchRequest getQueryFieldFromLidvid (String lidvid, String field, String es_index)
static public SearchRequest getqueryfieldfromlidvid (String lidvid, String field, String es_index)
{
List<String> fields = new ArrayList<String>(), lidvids = new ArrayList<String>();
Map<String,List<String>> kvps = new HashMap<String,List<String>>();
Expand Down Expand Up @@ -238,18 +187,21 @@ static public SearchRequest getQueryForKVPs (Map<String,List<String>> kvps, List

static public SearchRequest getQueryForKVPs (Map<String,List<String>> kvps, List<String> fields, String es_index, boolean term)
{
String[] aFields = new String[fields == null ? 0 : fields.size() + EntityProduct.JSON_PROPERTIES.length];
if (fields != null)
{
for (int i = 0 ; i < EntityProduct.JSON_PROPERTIES.length ; i++) aFields[i] = EntityProduct.JSON_PROPERTIES[i];
for (int i = 0 ; i < fields.size(); i++) aFields[i+EntityProduct.JSON_PROPERTIES.length] = ElasticSearchUtil.jsonPropertyToElasticProperty(fields.get(i));
}

String[] exclude = excludes (fields);
String[] include = fields.toArray(new String[0]);

BoolQueryBuilder find_kvps = QueryBuilders.boolQuery();
SearchRequest request = new SearchRequest(es_index)
.source(new SearchSourceBuilder().query(find_kvps)
.fetchSource(fields == null ? DEFAULT_ALL_FIELDS : aFields, DEFAULT_BLOB));

.fetchSource(include, exclude));

log.info("****************************************");
log.info("**************** exclude");
for (String e : exclude) log.info("**************** " + e);
log.info("****************************************");
log.info("**************** include");
for (String i : include) log.info("**************** " + i);
log.info("****************************************");
for (Entry<String,List<String>> key : kvps.entrySet())
{
for (String value : key.getValue())
Expand Down
12 changes: 1 addition & 11 deletions service/src/main/java/gov/nasa/pds/api/engineering/elasticsearch/ElasticSearchUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@
import org.slf4j.LoggerFactory;

import gov.nasa.pds.api.engineering.elasticsearch.entities.EntityProduct;
import gov.nasa.pds.api.engineering.elasticsearch.entities.EntitytProductWithBlob;
import gov.nasa.pds.api.engineering.exceptions.UnsupportedElasticSearchProperty;
import gov.nasa.pds.model.Metadata;
import gov.nasa.pds.model.PdsProduct;
import gov.nasa.pds.api.model.xml.ProductWithXmlLabel;
import gov.nasa.pds.model.Reference;
import gov.nasa.pds.model.Summary;

Expand Down Expand Up @@ -146,15 +144,7 @@ static private PdsProduct addPropertiesFromESEntity(
product.setTargets(targets);

return product;
}

static public ProductWithXmlLabel ESentityProductToAPIProduct(EntitytProductWithBlob ep, URL baseURL) {
log.debug("convert ES object to API object with XML label");
ProductWithXmlLabel product = new ProductWithXmlLabel();
product.setLabelXml(ep.getPDS4XML());
return (ProductWithXmlLabel)addPropertiesFromESEntity(product, ep, baseURL);
}

}

static public PdsProduct ESentityProductToAPIProduct(EntityProduct ep, URL baseURL) {
log.debug("convert ES object to API object without XML label");
Expand Down
Loading

0 comments on commit 57f4aa1

Please sign in to comment.