From b143b677a2b6fbbad4fdc86edd9b0661bf4a2c3b Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Wed, 18 Dec 2024 17:41:34 -0800 Subject: [PATCH 1/5] intitial working version of though resource instance search, no paging, re #8 --- afrc/src/afrc/Search/SearchPage.vue | 5 +- afrc/templates/arches_urls.htm | 1 + afrc/urls.py | 15 +- afrc/views/search_api.py | 335 ++++++++++++++++++++++++++++ 4 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 afrc/views/search_api.py diff --git a/afrc/src/afrc/Search/SearchPage.vue b/afrc/src/afrc/Search/SearchPage.vue index d7e367a..e4e9154 100644 --- a/afrc/src/afrc/Search/SearchPage.vue +++ b/afrc/src/afrc/Search/SearchPage.vue @@ -39,8 +39,7 @@ watch(queryString, () => { function updateFilter(componentName: string, value: object) { console.log(value); // Test for an empty object - // eslint-disable-next-line @typescript-eslint/no-explicit-any - function isEmpty(value: any) { + function isEmpty(value: unknown) { if (value === null || value === undefined) { return true; } @@ -89,7 +88,7 @@ const doQuery = function () { const qs = new URLSearchParams(queryObj); - fetch(arches.urls.search_results + "?" + qs.toString()) + fetch(arches.urls["api-search"] + "?" + qs.toString()) .then((response) => response.json()) .then((data) => { console.log(data); diff --git a/afrc/templates/arches_urls.htm b/afrc/templates/arches_urls.htm index 5eef4d9..80eba82 100644 --- a/afrc/templates/arches_urls.htm +++ b/afrc/templates/arches_urls.htm @@ -7,6 +7,7 @@ {% block arches_urls %} {{ block.super }}
[0-9]+|\{z\})/(?P[0-9]+|\{x\})/(?P[0-9]+|\{y\}).pbf$", + re_path( + r"^api-reference-collection-mvt/(?P[0-9]+|\{z\})/(?P[0-9]+|\{x\})/(?P[0-9]+|\{y\}).pbf$", ReferenceCollectionMVT.as_view(), - name="api-reference-collection-mvt"), + name="api-reference-collection-mvt", + ), ] # Ensure Arches core urls are superseded by project-level urls diff --git a/afrc/views/search_api.py b/afrc/views/search_api.py new file mode 100644 index 0000000..8352157 --- /dev/null +++ b/afrc/views/search_api.py @@ -0,0 +1,335 @@ +""" +ARCHES - a program developed to inventory and manage immovable cultural heritage. +Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +import logging +import os +import json + +from django.views.generic import View +from django.contrib.gis.geos import GEOSGeometry +from django.core.cache import cache +from django.db import connection +from django.http import Http404 +from django.shortcuts import render +from django.utils.translation import get_language, gettext as _ +from django.db.models import Q, OuterRef, Subquery +from arches.app.models.models import ( + MapMarker, + GraphModel, + DDataType, + Widget, + ReportTemplate, + CardComponent, + Geocoder, + Node, + SearchExportHistory, + ResourceXResource, +) +from arches.app.search.components.search_results import get_localized_descriptor +from arches.app.search.mappings import RESOURCES_INDEX +from arches.app.models.concept import Concept, get_preflabel_from_conceptid +from arches.app.utils.response import JSONResponse, JSONErrorResponse +from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer +from arches.app.search.search_engine_factory import SearchEngineFactory +from arches.app.search.elasticsearch_dsl_builder import ( + Bool, + Match, + Query, + Ids, + Term, + Terms, + MaxAgg, + Aggregation, +) +from arches.app.search.search_export import SearchResultsExporter +from arches.app.search.time_wheel import TimeWheel +from arches.app.search.components.base import SearchFilterFactory +from arches.app.views.base import MapBaseManagerView +from arches.app.utils import permission_backend +from arches.app.utils.permission_backend import ( + get_nodegroups_by_perm, + user_is_resource_reviewer, +) +from arches.app.utils.decorators import group_required +import arches.app.utils.zip as zip_utils +import arches.app.utils.task_management as task_management +from arches.app.utils.data_management.resources.formats.htmlfile import HtmlWriter +import arches.app.tasks as tasks +from io import StringIO +from tempfile import NamedTemporaryFile +from arches.app.models.system_settings import settings + +logger = logging.getLogger(__name__) + + +def search_results(request, returnDsl=False): + search_filter_factory = SearchFilterFactory(request) + searchview_component_instance = search_filter_factory.get_searchview_instance() + + if not searchview_component_instance: + unavailable_searchview_name = search_filter_factory.get_searchview_name() + message = _("No search-view named {0}").format(unavailable_searchview_name) + return JSONErrorResponse( + _("Search Failed"), + message, + status=400, + ) + + try: + response_object, search_query_object = ( + searchview_component_instance.handle_search_results_query( + search_filter_factory, returnDsl + ) + ) + if returnDsl: + return search_query_object.pop("query") + else: + return response_object + except Exception as e: + message = _("There was an error retrieving the search results") + try: + message = e.args[0].get("message", message) + except: + logger.exception("Error retrieving search results:") + logger.exception(e) + + return JSONErrorResponse( + _("Search Failed"), + message, + status=500, + ) + + +class SearchAPI(View): + def get(self, request): + + base_resource_type_filter = [ + { + "graphid": "d6774bfc-b4b4-11ea-84f7-3af9d3b32b71", + "name": "Group", + "inverted": False, + } + ] + + current_page = request.GET.get("paging-filter", 1) + page_size = int(settings.SEARCH_ITEMS_PER_PAGE) + print(page_size) + + request_copy = request.GET.copy() + request_copy["resource-type-filter"] = json.dumps(base_resource_type_filter) + request.GET = request_copy + direct_results = search_results(request) + print(current_page * page_size) + print(direct_results["total_results"]) + + if direct_results["total_results"] >= current_page * page_size: + print("we have direct hits on collections") + return JSONResponse(content=search_results(request)) + else: + # we have no more direct hits on reference collections and we need to + # backfill with results of hits based on potential resources related to reference collections + # So first we need to search for resources that aren't reference collections and that match our search criteria + # then we take those resource instance ids and do a recursive search for any of those + # resources that might be related to reference collections + # and return a list of those reference collections + base_resource_type_filter[0]["inverted"] = True + + request_copy = request.GET.copy() + request_copy["resource-type-filter"] = json.dumps(base_resource_type_filter) + request_copy["paging-filter"] = 1 + request.GET = request_copy + backfill_results = search_results(request) + + # first page of hits of potentially related resources + resourceinstanceids = [ + hit["_source"]["resourceinstanceid"] + for hit in backfill_results["results"]["hits"]["hits"] + ] + + related_resource_ids = list( + search_relationships_via_ORM(resourceinstanceids, depth=3) + ) + + se = SearchEngineFactory().create() + query = Query(se, start=0, limit=30) + query.add_query(Ids(ids=related_resource_ids)) + results = query.search(index=RESOURCES_INDEX) + + descriptor_types = ("displaydescription", "displayname") + active_and_default_language_codes = (get_language(), settings.LANGUAGE_CODE) + for result in results["hits"]["hits"]: + for descriptor_type in descriptor_types: + descriptor = get_localized_descriptor( + result, descriptor_type, active_and_default_language_codes + ) + if descriptor: + print(descriptor) + result["_source"][descriptor_type] = descriptor["value"] + if descriptor_type == "displayname": + result["_source"]["displayname_language"] = descriptor[ + "language" + ] + else: + result["_source"][descriptor_type] = _("Undefined") + direct_results["results"]["hits"]["hits"] += results["hits"]["hits"] + direct_results["total_results"] += int(len(results["hits"]["hits"])) + return JSONResponse(direct_results) + + +def search_relationships_via_ORM( + resourceinstanceids=None, + target_graphid="d6774bfc-b4b4-11ea-84f7-3af9d3b32b71", + depth=1, +): + hits = set() + + # This is a placeholder for the ORM version of the search_relationships function + # This function should return a list of resourceinstanceids of reference collections + # that are related to the given list of resourceinstanceids + def get_related_resourceinstanceids(resourceinstanceids, depth=1): + depth -= 1 + to_crawl = set() + + # This is a placeholder for the ORM version of the get_related_resourceinstanceids function + # This function should return a list of resourceinstanceids of resources that are related to + # the given list of resourceinstanceids + instances_query = Q(resourceinstanceidfrom__in=resourceinstanceids) | Q( + resourceinstanceidto__in=resourceinstanceids + ) + + for res in ResourceXResource.objects.filter(instances_query).values_list( + "resourceinstanceidfrom", + "resourceinstancefrom_graphid", + "resourceinstanceidto", + "resourceinstanceto_graphid", + ): + if str(res[1]) != target_graphid: + to_crawl.add(res[0]) + else: + hits.add(res[0]) + + if str(res[3]) != target_graphid: + to_crawl.add(res[2]) + else: + hits.add(res[2]) + + if depth > 0: + get_related_resourceinstanceids(list(to_crawl), depth=depth) + + return hits + + return get_related_resourceinstanceids(resourceinstanceids, depth=depth) + + +def search_relationships( + resourceinstanceids=None, target_graphid="d6774bfc-b4b4-11ea-84f7-3af9d3b32b71" +): + with connection.cursor() as cursor: + sql = """ + WITH RECURSIVE resource_traversal_from(resourcexid, resourceid, graphid, depth) AS ( + -- Anchor member: start with the given list of starting resource IDs + SELECT + resource_x_resource.resourcexid, resourceinstanceidto AS resourceid, resourceinstanceto_graphid AS graphid, 0 AS depth + FROM + resource_x_resource + WHERE + resourceinstanceidfrom = ANY(%s::uuid[]) + + UNION ALL + + -- Recursive member: traverse the table bidirectionally + SELECT + resource_x_resource.resourcexid, resource_x_resource.resourceinstanceidto AS resourceid, resourceinstanceto_graphid AS graphid, rt.depth + 1 + FROM + resource_x_resource + INNER JOIN + resource_traversal_from rt + ON + resource_x_resource.resourceinstanceidfrom = rt.resourceid + WHERE + rt.graphid != %s::uuid + + ) CYCLE resourcexid SET is_cycle USING path + + SELECT DISTINCT resourceid + FROM resource_traversal_from + WHERE graphid = %s::uuid + AND DEPTH < 3 + + UNION ( + WITH RECURSIVE resource_traversal_to(resourcexid, resourceid, graphid, depth) AS ( + -- Anchor member: start with the given list of starting resource IDs + SELECT + resource_x_resource.resourcexid, resourceinstanceidfrom AS resourceid, resourceinstancefrom_graphid AS graphid, 0 AS depth + FROM + resource_x_resource + WHERE + resourceinstanceidto = ANY(%s::uuid[]) + + UNION ALL + + SELECT + resource_x_resource.resourcexid, resource_x_resource.resourceinstanceidfrom AS resourceid, resourceinstancefrom_graphid AS graphid, rt.depth + 1 + FROM + resource_x_resource + INNER JOIN + resource_traversal_to rt + ON + resource_x_resource.resourceinstanceidto = rt.resourceid + WHERE + rt.graphid != %s::uuid + + ) CYCLE resourcexid SET is_cycle USING path + + SELECT DISTINCT resourceid + FROM resource_traversal_to + WHERE graphid = %s::uuid + AND DEPTH < 3 + ) + """ + print( + sql + % ( + resourceinstanceids, + target_graphid, + target_graphid, + resourceinstanceids, + target_graphid, + target_graphid, + ) + ) + cursor.execute( + sql, + [ + resourceinstanceids, + target_graphid, + target_graphid, + resourceinstanceids, + target_graphid, + target_graphid, + ], + ) + hits = [] + # hits = [str(row[0]) for row in cursor.fetchall()] + for row in cursor.fetchall(): + hits.append(str(row[0])) + print(len(hits)) + return hits + + +# {"query": {"ids": {"values": ["fba9bdb3-29a6-3cc2-bd7e-2d3fa7a08c78"]}}, "start": 0, "limit": 0} From 5f1f1cacb0ed2e54382705f662272fe546bb0c76 Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Thu, 19 Dec 2024 11:13:31 -0800 Subject: [PATCH 2/5] styling updates, re #8 --- afrc/settings.py | 2 ++ afrc/src/afrc/Search/SearchPage.vue | 6 ++-- .../Search/components/SearchResultItem.vue | 28 ++++++++++++++----- afrc/views/search_api.py | 15 +++++----- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/afrc/settings.py b/afrc/settings.py index ab61b35..02ac058 100644 --- a/afrc/settings.py +++ b/afrc/settings.py @@ -421,6 +421,8 @@ # override this to permenantly display/hide the language switcher SHOW_LANGUAGE_SWITCH = len(LANGUAGES) > 1 +COLLECTIONS_GRAPHID = "3d461890-b4b5-11ea-84f7-3af9d3b32b71" + # Implement this class to associate custom documents to the ES resource index # See tests.views.search_tests.TestEsMappingModifier class for example # ES_MAPPING_MODIFIER_CLASSES = ["afrc.search.es_mapping_modifier.EsMappingModifier"] diff --git a/afrc/src/afrc/Search/SearchPage.vue b/afrc/src/afrc/Search/SearchPage.vue index e4e9154..34a7a80 100644 --- a/afrc/src/afrc/Search/SearchPage.vue +++ b/afrc/src/afrc/Search/SearchPage.vue @@ -320,13 +320,13 @@ aside { gap: 10px; } .facet-item { - font-size: 0.7rem; + font-size: 1rem; padding: 16px; border: 1px solid #ddd; text-align: center; cursor: pointer; - max-width: 11rem; - min-height: 11rem; + max-width: 15rem; + min-height: 15rem; } .facet-item.selected { background-color: #f0f8ff; diff --git a/afrc/src/afrc/Search/components/SearchResultItem.vue b/afrc/src/afrc/Search/components/SearchResultItem.vue index f4cdcc4..fba9f7c 100644 --- a/afrc/src/afrc/Search/components/SearchResultItem.vue +++ b/afrc/src/afrc/Search/components/SearchResultItem.vue @@ -39,19 +39,22 @@ function clearResult() {
-

{{ props.searchResult._source.displayname }}

-

+

+ {{ props.searchResult._source.displayname }} +
+
(North and Central America > United States > Missouri > Greene) -

-

+

+
{{ searchResult._source.displaydescription }} -

+
@@ -81,15 +86,17 @@ function clearResult() { height: 16rem; overflow: hidden; padding-inline-start: 10px; + padding: 15px; } .result h2 { margin: 0 0 10px; font-size: 1.2rem; } .result .breadcrumb { - color: #888; - font-size: 0.9rem; + color: #415790; + font-size: 1.1rem; margin-bottom: 10px; + padding: unset; } .result .image-placeholder { width: 16rem; @@ -97,6 +104,13 @@ function clearResult() { min-width: 16rem; background-color: #eee; } +.result-displayname { + font-size: 1.5rem; + font-weight: bold; +} +.scope-note { + font-size: 1.2rem; +} .actions { display: flex; gap: 10px; diff --git a/afrc/views/search_api.py b/afrc/views/search_api.py index 8352157..2b59381 100644 --- a/afrc/views/search_api.py +++ b/afrc/views/search_api.py @@ -120,8 +120,7 @@ def get(self, request): base_resource_type_filter = [ { - "graphid": "d6774bfc-b4b4-11ea-84f7-3af9d3b32b71", - "name": "Group", + "graphid": settings.COLLECTIONS_GRAPHID, "inverted": False, } ] @@ -162,7 +161,11 @@ def get(self, request): ] related_resource_ids = list( - search_relationships_via_ORM(resourceinstanceids, depth=3) + search_relationships_via_ORM( + resourceinstanceids, + target_graphid=settings.COLLECTIONS_GRAPHID, + depth=3, + ) ) se = SearchEngineFactory().create() @@ -193,7 +196,7 @@ def get(self, request): def search_relationships_via_ORM( resourceinstanceids=None, - target_graphid="d6774bfc-b4b4-11ea-84f7-3af9d3b32b71", + target_graphid=None, depth=1, ): hits = set() @@ -236,9 +239,7 @@ def get_related_resourceinstanceids(resourceinstanceids, depth=1): return get_related_resourceinstanceids(resourceinstanceids, depth=depth) -def search_relationships( - resourceinstanceids=None, target_graphid="d6774bfc-b4b4-11ea-84f7-3af9d3b32b71" -): +def search_relationships(resourceinstanceids=None, target_graphid=None): with connection.cursor() as cursor: sql = """ WITH RECURSIVE resource_traversal_from(resourcexid, resourceid, graphid, depth) AS ( From 293471daec8f76ba26612981d37176a23e487c92 Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Thu, 19 Dec 2024 11:14:37 -0800 Subject: [PATCH 3/5] don't set default search graphid, re #8 --- afrc/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afrc/settings.py b/afrc/settings.py index 02ac058..4f5c315 100644 --- a/afrc/settings.py +++ b/afrc/settings.py @@ -421,7 +421,7 @@ # override this to permenantly display/hide the language switcher SHOW_LANGUAGE_SWITCH = len(LANGUAGES) > 1 -COLLECTIONS_GRAPHID = "3d461890-b4b5-11ea-84f7-3af9d3b32b71" +COLLECTIONS_GRAPHID = "" # Implement this class to associate custom documents to the ES resource index # See tests.views.search_tests.TestEsMappingModifier class for example From ac5d32ba42529b7934208ff1935e34a1751bd1e0 Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Thu, 19 Dec 2024 11:25:16 -0800 Subject: [PATCH 4/5] clean up imports, re #8 --- afrc/views/search_api.py | 55 +++++----------------------------------- 1 file changed, 7 insertions(+), 48 deletions(-) diff --git a/afrc/views/search_api.py b/afrc/views/search_api.py index 2b59381..9d7a51e 100644 --- a/afrc/views/search_api.py +++ b/afrc/views/search_api.py @@ -17,62 +17,21 @@ """ import logging -import os import json from django.views.generic import View -from django.contrib.gis.geos import GEOSGeometry -from django.core.cache import cache from django.db import connection -from django.http import Http404 -from django.shortcuts import render from django.utils.translation import get_language, gettext as _ -from django.db.models import Q, OuterRef, Subquery -from arches.app.models.models import ( - MapMarker, - GraphModel, - DDataType, - Widget, - ReportTemplate, - CardComponent, - Geocoder, - Node, - SearchExportHistory, - ResourceXResource, -) +from django.db.models import Q + +from arches.app.models.models import ResourceXResource +from arches.app.models.system_settings import settings +from arches.app.search.components.base import SearchFilterFactory from arches.app.search.components.search_results import get_localized_descriptor +from arches.app.search.elasticsearch_dsl_builder import Query, Ids from arches.app.search.mappings import RESOURCES_INDEX -from arches.app.models.concept import Concept, get_preflabel_from_conceptid -from arches.app.utils.response import JSONResponse, JSONErrorResponse -from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.search.search_engine_factory import SearchEngineFactory -from arches.app.search.elasticsearch_dsl_builder import ( - Bool, - Match, - Query, - Ids, - Term, - Terms, - MaxAgg, - Aggregation, -) -from arches.app.search.search_export import SearchResultsExporter -from arches.app.search.time_wheel import TimeWheel -from arches.app.search.components.base import SearchFilterFactory -from arches.app.views.base import MapBaseManagerView -from arches.app.utils import permission_backend -from arches.app.utils.permission_backend import ( - get_nodegroups_by_perm, - user_is_resource_reviewer, -) -from arches.app.utils.decorators import group_required -import arches.app.utils.zip as zip_utils -import arches.app.utils.task_management as task_management -from arches.app.utils.data_management.resources.formats.htmlfile import HtmlWriter -import arches.app.tasks as tasks -from io import StringIO -from tempfile import NamedTemporaryFile -from arches.app.models.system_settings import settings +from arches.app.utils.response import JSONResponse, JSONErrorResponse logger = logging.getLogger(__name__) From 07c8372090ddf685df7d9e690c52c5106f79888e Mon Sep 17 00:00:00 2001 From: Alexei Peters Date: Thu, 9 Jan 2025 19:13:36 -0800 Subject: [PATCH 5/5] removed unused functions, re #8 --- afrc/src/afrc/Search/SearchPage.vue | 37 ------------------- .../Search/components/SimpleSearchFilter.vue | 4 ++ 2 files changed, 4 insertions(+), 37 deletions(-) diff --git a/afrc/src/afrc/Search/SearchPage.vue b/afrc/src/afrc/Search/SearchPage.vue index 34a7a80..cb74c6f 100644 --- a/afrc/src/afrc/Search/SearchPage.vue +++ b/afrc/src/afrc/Search/SearchPage.vue @@ -96,43 +96,6 @@ const doQuery = function () { resultsCount.value = data.total_results; resultsSelected.value = []; }); - - // self.updateRequest = $.ajax({ - // type: "GET", - // url: arches.urls.search_results, - // data: queryObj, - // context: this, - // success: function(response) { - // _.each(this.sharedStateObject.searchResults, function(value, key, results) { - // if (key !== "timestamp") { - // delete this.sharedStateObject.searchResults[key]; - // } - // }, this); - // _.each(response, function(value, key, response) { - // if (key !== "timestamp") { - // this.sharedStateObject.searchResults[key] = value; - // } - // }, this); - // this.sharedStateObject.searchResults.timestamp(response.timestamp); - // this.sharedStateObject.userIsReviewer(response.reviewer); - // this.sharedStateObject.userid(response.userid); - // this.sharedStateObject.total(response.total_results); - // this.sharedStateObject.hits(response.results.hits.hits.length); - // this.sharedStateObject.alert(false); - // }, - // error: function(response, status, error) { - // const alert = new AlertViewModel("ep-alert-red", arches.translations.requestFailed.title, response.responseJSON?.message); - // if(self.updateRequest.statusText !== "abort"){ - // this.alert(alert); - // } - // this.sharedStateObject.loading(false); - // }, - // complete: function(request, status) { - // self.updateRequest = undefined; - // window.history.pushState({}, "", "?" + $.param(queryObj).split("+").join("%20")); - // this.sharedStateObject.loading(false); - // } - // }); }; async function fetchSystemMapData() { diff --git a/afrc/src/afrc/Search/components/SimpleSearchFilter.vue b/afrc/src/afrc/Search/components/SimpleSearchFilter.vue index 1f8ecaa..04ac527 100644 --- a/afrc/src/afrc/Search/components/SimpleSearchFilter.vue +++ b/afrc/src/afrc/Search/components/SimpleSearchFilter.vue @@ -113,6 +113,10 @@ const updateQuery = function () { background-color: lightgray; font-family: Arial, Helvetica, sans-serif; } +.p-autocomplete-token-label { + font-size: 1.3rem; +} + :deep(.autocomplete-input) { height: 3rem; font-size: 1.5rem;