diff --git a/.coveragerc b/.coveragerc index e06d635ad89..3d0c8f70546 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,29 @@ -[report] +[run] +source = + arches/ + omit = */python?.?/* */models/migrations/* - -show_missing = true \ No newline at end of file + */settings*.py + */urls.py + */wsgi.py + */celery.py + */__init__.py + +data_file = coverage/python/.coverage + +[report] +show_missing = true + +exclude_lines = + pragma: no cover + +[html] +directory = coverage/python/htmlcov + +[xml] +output = coverage/python/coverage.xml + +[json] +output = coverage/python/coverage.json diff --git a/.dockerignore b/.dockerignore index 4a8caf89576..3ba92533947 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,4 +18,4 @@ arches/settings_local.pyc elasticsearch-5.2.1 virtualenv arches/app/media/packages -arches/app/media/node_modules +node_modules diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 00d347c90f6..00000000000 --- a/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -!.eslintrc.js \ No newline at end of file diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index 0dd31b80fbe..00000000000 --- a/.eslintrc.js +++ /dev/null @@ -1,64 +0,0 @@ -module.exports = { - "extends": [ - "eslint:recommended", - 'plugin:@typescript-eslint/recommended', - 'plugin:vue/vue3-recommended', - ], - "root": true, - "env": { - "browser": true, - "es6": true, - "node": true - }, - "parser": "vue-eslint-parser", - "parserOptions": { - "ecmaVersion": 11, - "sourceType": "module", - "requireConfigFile": false, - "parser": { - "ts": "@typescript-eslint/parser" - } - }, - "globals": { - "define": false, - "require": false, - "window": false, - "console": false, - "history": false, - "location": false, - "Promise": false, - "setTimeout": false, - "URL": false, - "URLSearchParams": false, - "fetch": false - }, - "ignorePatterns": [".eslintrc.js", "**/media/plugins/*"], - "rules": { - "semi": ["error", "always"], - }, - "overrides": [ - { - "files": [ "*.vue" ], - "rules": { - "vue/html-indent": [2, 4], - } - }, - { - "files": [ "*.js" ], - "rules": { - "indent": ["error", 4], - "space-before-function-paren": ["error", "never"], - "no-extra-boolean-cast": 0, // 0=silence, 1=warning, 2=error - // allow async-await - 'generator-star-spacing': 'off', - // allow debugger during development - 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', - 'no-unused-vars': [1, { - argsIgnorePattern: '^_' - }], - "camelcase": [1, {"properties": "always"}], - } - } - ] -}; - \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 39645f99873..aa77801b40c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,14 +6,15 @@ on: workflow_dispatch: jobs: - build: + build_feature_branch: runs-on: ubuntu-latest + services: postgres: image: postgis/postgis:13-3.0 env: POSTGRES_PASSWORD: postgis - POSTGRES_DB: arches + POSTGRES_DB: ${{ github.event.repository.name }} ports: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 @@ -25,6 +26,7 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -37,14 +39,116 @@ jobs: sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev libxslt-dev echo Postgres and ES dependencies installed - - name: Install python packages + - name: Install Python packages run: | python -m pip install --upgrade pip pip install . - pip install -r arches/install/requirements.txt - pip install -r arches/install/requirements_dev.txt + pip install -r ${{ github.event.repository.name }}/install/requirements.txt + pip install -r ${{ github.event.repository.name }}/install/requirements_dev.txt echo Python packages installed + - uses: ankane/setup-elasticsearch@v1 + with: + elasticsearch-version: 8 + + - name: Webpack frontend files + run: | + echo "Checking for yarn.lock file..." + if [ -f yarn.lock ]; then + echo "Removing yarn.lock due to yarn v1 package resolution issues" + echo "https://github.com/iarna/wide-align/issues/63" + rm yarn.lock + else + echo "yarn.lock not found, skipping remove." + fi + + echo "Checking for package.json..." + if [ -f package.json ]; then + echo "package.json found, building static bundle." + yarn && yarn build_test + else + echo "package.json not found, skipping yarn commands." + fi + + - name: Run frontend tests + run: | + yarn vitest + mv coverage/frontend/coverage.xml feature_branch_frontend_coverage.xml + + - name: Check for missing migrations + run: | + python manage.py makemigrations --check + + - name: Ensure previous Python coverage data is erased + run: | + coverage erase + + - name: Run Python unit tests + run: | + python -W default::DeprecationWarning -m coverage run manage.py test tests --settings="tests.test_settings" + + - name: Generate Python report coverage + run: | + coverage report + coverage json + mv coverage/python/coverage.json feature_branch_python_coverage.json + + - name: Upload frontend coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: feature-branch-frontend-coverage-report + path: feature_branch_frontend_coverage.xml + overwrite: true + + - name: Upload Python coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: feature-branch-python-coverage-report + path: feature_branch_python_coverage.json + overwrite: true + + build_target_branch: + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: ${{ github.event.repository.name }} + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + + - name: Install Java, GDAL, and other system dependencies + run: | + sudo apt update + sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev libxslt-dev + echo Postgres and ES dependencies installed + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install . + pip install -r ${{ github.event.repository.name }}/install/requirements.txt + pip install -r ${{ github.event.repository.name }}/install/requirements_dev.txt + echo Python packages installed - uses: ankane/setup-elasticsearch@v1 with: @@ -52,19 +156,249 @@ jobs: - name: Webpack frontend files run: | - echo "Removing yarn.lock due to yarn v1 package resolution issues" - echo "https://github.com/iarna/wide-align/issues/63" - rm yarn.lock - yarn && yarn build_test + echo "Checking for yarn.lock file..." + if [ -f yarn.lock ]; then + echo "Removing yarn.lock due to yarn v1 package resolution issues" + echo "https://github.com/iarna/wide-align/issues/63" + rm yarn.lock + else + echo "yarn.lock not found, skipping remove." + fi + + echo "Checking for package.json..." + if [ -f package.json ]; then + echo "package.json found, building static bundle." + yarn && yarn build_test + else + echo "package.json not found, skipping yarn commands." + fi + + - name: Run frontend tests + run: | + if [ -f vitest.config.json ]; then + yarn vitest + mv coverage/frontend/coverage.xml target_branch_frontend_coverage.xml + else + echo "Unable to find vitest config. Skipping frontend tests." + fi - name: Check for missing migrations run: | python manage.py makemigrations --check - - name: Run Arches unit tests + - name: Ensure previous Python coverage data is erased + run: | + coverage erase + + - name: Run Python unit tests run: | python -W default::DeprecationWarning -m coverage run manage.py test tests --settings="tests.test_settings" - - name: Report coverage + - name: Generate Python report coverage run: | coverage report + coverage json + + # handles older target branch + if [ -f coverage/python/coverage.json ]; then + mv coverage/python/coverage.json target_branch_python_coverage.json + else + mv coverage.json target_branch_python_coverage.json + fi + + - name: Upload frontend coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: target-branch-frontend-coverage-report + path: target_branch_frontend_coverage.xml + overwrite: true + + - name: Upload Python coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: target-branch-python-coverage-report + path: target_branch_python_coverage.json + overwrite: true + + check_frontend_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch frontend coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-frontend-coverage-report + path: . + + - name: Extract feature branch frontend coverage data + shell: pwsh + run: | + [xml]$xml = Get-Content feature_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $feature_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $feature_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "feature_branch_frontend_coverage=$feature_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Download target branch frontend coverage report artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: target-branch-frontend-coverage-report + path: . + + - name: Check if target branch frontend coverage report artifact exists + run: | + if [ -f target_branch_frontend_coverage.xml ]; then + echo "target_branch_frontend_coverage_artifact_exists=true" >> $GITHUB_ENV + else + echo "Target branch coverage not found. Defaulting to 0% coverage." + echo "target_branch_frontend_coverage_artifact_exists=false" >> $GITHUB_ENV + fi + + - name: Extract target branch frontend coverage data + if: ${{ env.target_branch_frontend_coverage_artifact_exists == 'true' }} + shell: pwsh + run: | + [xml]$xml = Get-Content target_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $target_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $target_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "target_branch_frontend_coverage=$target_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Compare frontend feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_frontend_coverage=${feature_branch_frontend_coverage} + target_branch_frontend_coverage=${target_branch_frontend_coverage:-0.0} + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_frontend_coverage" -v target="$target_branch_frontend_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_frontend_coverage% to $feature_branch_frontend_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_frontend_coverage%) >= Target branch coverage ($target_branch_frontend_coverage%)." + fi + + check_python_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-python-coverage-report + path: . + + - name: Download target branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: target-branch-python-coverage-report + path: . + + - name: Compare Python feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_python_coverage=$(cat feature_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + target_branch_python_coverage=$(cat target_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_python_coverage" -v target="$target_branch_python_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_python_coverage% to $feature_branch_python_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_python_coverage%) >= Target branch coverage ($target_branch_python_coverage%)." + fi diff --git a/.gitignore b/.gitignore index 2a7f3e707eb..3c95bc55e37 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,7 @@ arches/uploadedfiles arches/settings_local.py tests/settings_local.py arches/logs/authority_file_errors.txt -.coverage -coverage.* +coverage/ arches.log .atom-build.json .atom-build.yml @@ -23,7 +22,6 @@ arches/Net_files .idea arches/app/media/bower_components arches/app/media/packages -arches/app/media/node_modules node_modules arches/tileserver/cache docs/_build @@ -43,4 +41,4 @@ pip-wheel-metadata *.code-workspace webpack-stats.json .DS_STORE -CACHE \ No newline at end of file +CACHE diff --git a/.yarnrc b/.yarnrc deleted file mode 100644 index dd5e7bd9d14..00000000000 --- a/.yarnrc +++ /dev/null @@ -1 +0,0 @@ ---modules-folder arches/app/media/node_modules \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 242b4e7a960..214a0789829 100644 --- a/Dockerfile +++ b/Dockerfile @@ -90,7 +90,7 @@ COPY ./arches/install/package.json ${ARCHES_ROOT}/arches/install/package.json COPY ./arches/install/.yarnrc ${ARCHES_ROOT}/arches/install/.yarnrc COPY ./arches/install/yarn.lock ${ARCHES_ROOT}/arches/install/yarn.lock WORKDIR ${ARCHES_ROOT}/arches/install -RUN mkdir -p ${ARCHES_ROOT}/arches/app/media/node_modules +RUN mkdir -p ${ARCHES_ROOT}/node_modules RUN yarn install ## Install virtualenv diff --git a/arches/app/const.py b/arches/app/const.py index 9d422b27450..b93022b59af 100644 --- a/arches/app/const.py +++ b/arches/app/const.py @@ -20,3 +20,4 @@ class ExtensionType(Enum): ETL_MODULES = "etl_modules" FUNCTIONS = "functions" SEARCH_COMPONENTS = "search_components" + PERMISSIONS_FRAMEWORKS = "permissions" diff --git a/arches/app/datatypes/base.py b/arches/app/datatypes/base.py index 99de186407e..13f1ba88676 100644 --- a/arches/app/datatypes/base.py +++ b/arches/app/datatypes/base.py @@ -109,6 +109,11 @@ def get_map_source(self, node=None, preview=False): if node is None: return None source_config = {"type": "vector", "tiles": [tileserver_url]} + node_config = json.loads(node.config.value) + for prop in ("minzoom", "maxzoom"): + if prop in node_config: + source_config[prop] = node_config[prop] + count = None if preview == True: count = models.TileModel.objects.filter(nodegroup_id=node.nodegroup_id, data__has_key=str(node.nodeid)).count() @@ -202,6 +207,8 @@ def get_tile_data(self, tile): return provisionaledits[userid]["value"] elif not data: logger.exception(_("Tile has no authoritative or provisional data")) + else: + return data def get_display_value(self, tile, node, **kwargs): diff --git a/arches/app/datatypes/concept_types.py b/arches/app/datatypes/concept_types.py index e92b7173faa..ad08c2a6e9b 100644 --- a/arches/app/datatypes/concept_types.py +++ b/arches/app/datatypes/concept_types.py @@ -37,7 +37,7 @@ def lookup_label(self, label, collectionid): ret = label collection_values = self.collection_lookup[collectionid] for concept in collection_values: - if label == concept[1]: + if concept[1] in (label, label.strip()): ret = concept[2] return ret diff --git a/arches/app/datatypes/datatypes.py b/arches/app/datatypes/datatypes.py index 66bfc990dda..aee304aa380 100644 --- a/arches/app/datatypes/datatypes.py +++ b/arches/app/datatypes/datatypes.py @@ -913,33 +913,39 @@ def default_es_mapping(self): class GeojsonFeatureCollectionDataType(BaseDataType): + def __init__(self, model=None): + super(GeojsonFeatureCollectionDataType, self).__init__(model=model) + self.geo_utils = GeoUtils() + def validate(self, value, row_number=None, source=None, node=None, nodeid=None, strict=False, **kwargs): errors = [] - coord_limit = 1500 - coordinate_count = 0 + max_bytes = 32766 # max bytes allowed by Lucene + byte_count = 0 + byte_count += len(str(value).encode("UTF-8")) + + def validate_geom_byte_size_can_be_reduced(feature_collection): + try: + if len(feature_collection['features']) > 0: + feature_geom = GEOSGeometry(JSONSerializer().serialize(feature_collection['features'][0]['geometry'])) + current_precision = abs(self.find_num(feature_geom.coords)) + feature_collection = self.geo_utils.reduce_precision(feature_collection, current_precision) + except ValueError: + message = _("Geojson byte size exceeds Lucene 32766 limit.") + title = _("Geometry Size Exceeds Elasticsearch Limit") + errors.append( + { + "type": "ERROR", + "message": "datatype: {0} {1} - {2}. {3}.".format( + self.datatype_model.datatype, source, message, "This data was not imported." + ), + "title": title, + } + ) - def validate_geom(geom, coordinate_count=0): + def validate_geom_bbox(geom): try: - coordinate_count += geom.num_coords bbox = Polygon(settings.DATA_VALIDATION_BBOX) - if coordinate_count > coord_limit: - message = _( - "Geometry has too many coordinates for Elasticsearch ({0}), \ - Please limit to less then {1} coordinates of 5 digits of precision or less.".format( - coordinate_count, coord_limit - ) - ) - title = _("Geometry Too Many Coordinates for ES") - errors.append( - { - "type": "ERROR", - "message": "datatype: {0} value: {1} {2} - {3}. {4}".format( - self.datatype_model.datatype, value, source, message, "This data was not imported." - ), - "title": title, - } - ) - + if bbox.contains(geom) == False: message = _( "Geometry does not fall within the bounding box of the selected coordinate system. \ @@ -969,12 +975,17 @@ def validate_geom(geom, coordinate_count=0): ) if value is not None: + if byte_count > max_bytes: + validate_geom_byte_size_can_be_reduced(value) for feature in value["features"]: try: geom = GEOSGeometry(JSONSerializer().serialize(feature["geometry"])) - validate_geom(geom, coordinate_count) + if geom.valid: + validate_geom_bbox(geom) + else: + raise Exception except Exception: - message = _("Unable to serialize some geometry features") + message = _("Unable to serialize some geometry features.") title = _("Unable to Serialize Geometry") error_message = self.create_error_message(value, source, row_number, message, title) errors.append(error_message) @@ -992,7 +1003,7 @@ def clean(self, tile, nodeid): def transform_value_for_tile(self, value, **kwargs): if "format" in kwargs and kwargs["format"] == "esrijson": - arches_geojson = GeoUtils().arcgisjson_to_geojson(value) + arches_geojson = self.geo_utils.arcgisjson_to_geojson(value) else: try: geojson = json.loads(value) @@ -1003,20 +1014,14 @@ def transform_value_for_tile(self, value, **kwargs): else: raise TypeError except (json.JSONDecodeError, KeyError, TypeError): - arches_geojson = {} - arches_geojson["type"] = "FeatureCollection" - arches_geojson["features"] = [] try: geometry = GEOSGeometry(value, srid=4326) if geometry.geom_type == "GeometryCollection": - for geom in geometry: - arches_json_geometry = {} - arches_json_geometry["geometry"] = JSONDeserializer().deserialize(GEOSGeometry(geom, srid=4326).json) - arches_json_geometry["type"] = "Feature" - arches_json_geometry["id"] = str(uuid.uuid4()) - arches_json_geometry["properties"] = {} - arches_geojson["features"].append(arches_json_geometry) + arches_geojson = self.geo_utils.convert_geos_geom_collection_to_feature_collection(geometry) else: + arches_geojson = {} + arches_geojson["type"] = "FeatureCollection" + arches_geojson["features"] = [] arches_json_geometry = {} arches_json_geometry["geometry"] = JSONDeserializer().deserialize(geometry.json) arches_json_geometry["type"] = "Feature" @@ -1041,7 +1046,24 @@ def update(self, tile, data, nodeid=None, action=None): updated_data = tile.data[nodeid] return updated_data + def find_num(self, current_item): + if len(current_item) and isinstance(current_item[0], float): + return decimal.Decimal(str(current_item[0])).as_tuple().exponent + else: + return self.find_num(current_item[0]) + def append_to_document(self, document, nodevalue, nodeid, tile, provisional=False): + max_bytes = 32766 # max bytes allowed by Lucene + byte_count = 0 + byte_count += len(str(nodevalue).encode("UTF-8")) + + if len(nodevalue['features']) > 0: + feature_geom = GEOSGeometry(JSONSerializer().serialize(nodevalue['features'][0]['geometry'])) + current_precision = abs(self.find_num(feature_geom.coords)) + + if byte_count > max_bytes and current_precision: + nodevalue = self.geo_utils.reduce_precision(nodevalue, current_precision) + document["geometries"].append({"geom": nodevalue, "nodegroup_id": tile.nodegroup_id, "provisional": provisional, "tileid": tile.pk}) bounds = self.get_bounds_from_value(nodevalue) if bounds is not None: diff --git a/arches/app/etl_modules/__init__.py b/arches/app/etl_modules/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/arches/app/etl_modules/bulk_data_deletion.py b/arches/app/etl_modules/bulk_data_deletion.py index 462eac85f00..c722fb648e7 100644 --- a/arches/app/etl_modules/bulk_data_deletion.py +++ b/arches/app/etl_modules/bulk_data_deletion.py @@ -1,6 +1,7 @@ from datetime import datetime import json import logging +import pyprind import uuid from django.contrib.auth.models import User from django.core.exceptions import ValidationError @@ -129,19 +130,34 @@ def get_sample_data(self, nodegroup_id, resourceids): return sample_data[0:5] - def delete_resources(self, userid, loadid, graphid, resourceids): + def delete_resources(self, userid, loadid, graphid=None, resourceids=None, verbose=False): result = {"success": False} - user = User.objects.get(id=userid) + deleted_count = 0 + user = User.objects.get(id=userid) if userid else {} try: - if resourceids and graphid: - resources = Resource.objects.filter(graph_id=graphid).filter(pk__in=resourceids) + if resourceids: + resources = Resource.objects.filter(pk__in=resourceids) elif graphid: resources = Resource.objects.filter(graph_id=graphid) - elif resourceids: - resources = Resource.objects.filter(pk__in=resourceids) + else: + result["message"] = _("Unable to bulk delete resources as no graphid or resourceids specified.") + result["deleted_count"] = 0 + return result + + deleted_count = resources.count() + + if verbose is True: + bar = pyprind.ProgBar(deleted_count) for resource in resources.iterator(chunk_size=2000): resource.delete(user=user, index=False, transaction_id=loadid) + if verbose is True: + bar.update() + + if verbose is True: + print(bar) result["success"] = True + result["deleted_count"] = deleted_count + result["message"] = _("Successfully deleted {} resources").format(str(deleted_count)) except Exception as e: logger.exception(e) result["message"] = _("Unable to delete resources: {}").format(str(e)) @@ -168,7 +184,7 @@ def delete_tiles(self, userid, loadid, nodegroupid, resourceids): return result - def index_resource_deletion(self, loadid, resourceids): + def index_resource_deletion(self, loadid, resourceids=None): if not resourceids: with connection.cursor() as cursor: cursor.execute( @@ -288,7 +304,7 @@ def run_bulk_task(self, userid, loadid, graph_id, nodegroup_id, resourceids): if nodegroup_id: deleted = self.delete_tiles(userid, loadid, nodegroup_id, resourceids) elif graph_id or resourceids: - deleted = self.delete_resources(userid, loadid, graph_id, resourceids) + deleted = self.delete_resources(userid, loadid, graphid=graph_id, resourceids=resourceids) with connection.cursor() as cursor: if deleted["success"]: @@ -341,7 +357,7 @@ def run_bulk_task(self, userid, loadid, graph_id, nodegroup_id, resourceids): if nodegroup_id: self.index_tile_deletion(loadid) else: - self.index_resource_deletion(loadid, resourceids) + self.index_resource_deletion(loadid, resourceids=resourceids) except Exception as e: logger.exception(e) with connection.cursor() as cursor: diff --git a/arches/app/etl_modules/import_single_csv.py b/arches/app/etl_modules/import_single_csv.py index 319160e7538..33341935363 100644 --- a/arches/app/etl_modules/import_single_csv.py +++ b/arches/app/etl_modules/import_single_csv.py @@ -10,7 +10,7 @@ from django.db.models.functions import Lower from django.utils.translation import gettext as _ from arches.app.datatypes.datatypes import DataTypeFactory -from arches.app.models.models import GraphModel, Node, NodeGroup +from arches.app.models.models import GraphModel, Node, NodeGroup, ETLModule from arches.app.models.system_settings import settings import arches.app.tasks as tasks from arches.app.utils.betterJSONSerializer import JSONSerializer @@ -26,6 +26,7 @@ def __init__(self, request=None, loadid=None): self.loadid = request.POST.get("load_id") if request else loadid self.userid = request.user.id if request else None self.moduleid = request.POST.get("module") if request else None + self.config = ETLModule.objects.get(pk=self.moduleid).config if self.moduleid else {} self.datatype_factory = DataTypeFactory() self.node_lookup = {} self.blank_tile_lookup = {} @@ -153,7 +154,7 @@ def write(self, request): temp_dir = os.path.join(settings.UPLOADED_FILES_DIR, "tmp", self.loadid) csv_file_path = os.path.join(temp_dir, csv_file_name) csv_size = default_storage.size(csv_file_path) # file size in byte - use_celery_threshold = 500 # 500 bytes + use_celery_threshold = self.config.get("celeryByteSizeLimit", 500) if csv_size > use_celery_threshold: response = self.run_load_task_async(request, self.loadid) diff --git a/arches/app/media/css/abstracts/_functions.scss b/arches/app/media/css/abstracts/_functions.scss index eb7d26c2f33..9fd26918952 100644 --- a/arches/app/media/css/abstracts/_functions.scss +++ b/arches/app/media/css/abstracts/_functions.scss @@ -5,7 +5,7 @@ // to this list, ordered by width. For examples: (mobile, tablet, desktop). // $mq-show-breakpoints: (mobile, mobileLandscape, tablet, desktop, wide); -//@import '../../node_modules/sass-mq/mq.import'; +//@import url(node_modules/sass-mq/mq.import); /// Responsive breakpoint manager /// @access public diff --git a/arches/app/media/css/arches.scss b/arches/app/media/css/arches.scss index 64c616da196..ff3492aa206 100644 --- a/arches/app/media/css/arches.scss +++ b/arches/app/media/css/arches.scss @@ -9662,11 +9662,13 @@ ul.pagination { .search-listing-footer { display: flex; - height: 40px; + height: fit-content; font-size: 1.1rem; - padding: 10px 10px 0px 10px; + padding: 10px 10px 10px 10px; background: #f5f5f5; border-top: 1px solid #ddd; + flex-flow: row wrap; + row-gap: 10px; a { margin-top: -5px; diff --git a/arches/app/media/js/utils/create-vue-application.js b/arches/app/media/js/utils/create-vue-application.js index f182484e26f..9fbc526a43b 100644 --- a/arches/app/media/js/utils/create-vue-application.js +++ b/arches/app/media/js/utils/create-vue-application.js @@ -6,6 +6,7 @@ import FocusTrap from 'primevue/focustrap'; import StyleClass from 'primevue/styleclass'; import ToastService from 'primevue/toastservice'; import Tooltip from 'primevue/tooltip'; +import arches from 'arches'; import { createApp } from 'vue'; import { createGettext } from "vue3-gettext"; @@ -24,7 +25,7 @@ export default async function createVueApplication(vueComponent){ * TODO: cbyrd #10501 - we should add an event listener that will re-fetch i18n data * and rebuild the app when a specific event is fired from the LanguageSwitcher component. **/ - return fetch('/api/get_frontend_i18n_data').then(resp => { + return fetch(arches.urls.api_get_frontend_i18n_data).then(resp => { if (!resp.ok) { throw new Error(resp.statusText); } diff --git a/arches/app/media/js/viewmodels/provisional-tile.js b/arches/app/media/js/viewmodels/provisional-tile.js index fac54a57aec..4aff7f0e02b 100644 --- a/arches/app/media/js/viewmodels/provisional-tile.js +++ b/arches/app/media/js/viewmodels/provisional-tile.js @@ -102,17 +102,17 @@ define([ if (self.selectedProvisionalEdit() != val) { self.selectedProvisionalEdit(val); koMapping.fromJS(val['value'], self.selectedTile().data); - self.selectedTile()._tileData.valueHasMutated(); + self.selectedTile().parent.params.handlers['tile-reset'].forEach(handler => handler(self.selectedTile())); self.selectedTile().parent.widgets().forEach( function(w){ var defaultconfig = w.widgetLookup[w.widget_id()].defaultconfig; - if (JSON.parse(defaultconfig).rerender === true && self.selectedTile().parent.allowProvisionalEditRerender() === true) { - self.selectedTile().parent.widgets()[0].label.valueHasMutated(); + if (defaultconfig.rerender === true && self.selectedTile().parent.allowProvisionalEditRerender() === true) { + w.label.valueHasMutated(); } - if (self.selectedTile().parent.triggerUpdate) { - self.selectedTile().parent.triggerUpdate(); - } }); + if (self.selectedTile().parent.triggerUpdate) { + self.selectedTile().parent.triggerUpdate(); + } } }; diff --git a/arches/app/media/js/viewmodels/resource-instance-select.js b/arches/app/media/js/viewmodels/resource-instance-select.js index 0715422a722..2e3d275986d 100644 --- a/arches/app/media/js/viewmodels/resource-instance-select.js +++ b/arches/app/media/js/viewmodels/resource-instance-select.js @@ -264,7 +264,7 @@ define([ inverseOntologyProperty = graph.config.inverseOntologyProperty; if (self.node && (!ontologyProperty || !inverseOntologyProperty) ) { - self.relationship(self.node.config.graphs()?.[0]?.useOntologyRelationship); + self.relationship(!!self.node.ontologyclass()); var ontologyProperties = self.node.config.graphs().find(function(nodeConfigGraph) { return nodeConfigGraph.graphid === graph.graphid; }); diff --git a/arches/app/media/js/viewmodels/tile.js b/arches/app/media/js/viewmodels/tile.js index 4d55e73ab45..3776e822cd4 100644 --- a/arches/app/media/js/viewmodels/tile.js +++ b/arches/app/media/js/viewmodels/tile.js @@ -84,6 +84,7 @@ define([ _.extend(this, { filter: filter, parent: params.card, + handlers: params.handlers, userisreviewer: params.userisreviewer, cards: _.filter(params.cards, function(card) { var nodegroup = _.find(ko.unwrap(params.graphModel.get('nodegroups')), function(group) { diff --git a/arches/app/media/js/views/rdm/modals/import-concept-form.js b/arches/app/media/js/views/rdm/modals/import-concept-form.js index a18b7f2fdd0..f4236063451 100644 --- a/arches/app/media/js/views/rdm/modals/import-concept-form.js +++ b/arches/app/media/js/views/rdm/modals/import-concept-form.js @@ -8,7 +8,7 @@ define(['jquery', 'backbone', 'arches', 'models/concept'], function($, Backbone, this.endpoint = this.$el.find('#sparql_endpoint').select2({ minimumResultsForSearch: -1 }); - this.$el.find('input.concept_import').select2({ + this.$el.find('select.concept_import').select2({ // multiple: false, // maximumselectionsize: 1, minimumInputLength: 2, @@ -16,23 +16,25 @@ define(['jquery', 'backbone', 'arches', 'models/concept'], function($, Backbone, ajax: { url: arches.urls.search_sparql_endpoint, dataType: 'json', - data: function(term, page) { + data: function(requestParams) { return { - terms: term, + terms: requestParams.term, endpoint: self.endpoint.val() }; }, - results: function(data, page) { + processResults: function(data, page) { + data.results.bindings.forEach((item)=> { + item.id = item.Subject.value; + return item; + }); return {results: data.results.bindings}; } }, - formatResult:function(result, container, query, escapeMarkup){ - var markup=[]; - window.Select2.util.markMatch(result.Term.value, query.term, markup, escapeMarkup); - if (!result.ScopeNote){ - result.ScopeNote = {'value': ''}; + templateResult:function(result, container, query, escapeMarkup){ + if (result.loading || result.children) { + return result.text; } - var formatedresult = '' + markup.join("") + ' - ' + result.Subject.value + '
(' + result.ScopeNote.value + ')
'; + var formatedresult = '' + result.Term.value + ' - ' + result?.Subject?.value + '
(' + result?.ScopeNote?.value + ')
'; return formatedresult; }, escapeMarkup: function(m) { return m; } diff --git a/arches/app/models/concept.py b/arches/app/models/concept.py index e90d8bc9a7f..2cb7d386b94 100644 --- a/arches/app/models/concept.py +++ b/arches/app/models/concept.py @@ -703,8 +703,8 @@ def get_child_edges( JOIN children c ON(c.conceptidto = valueto.conceptid) JOIN values valuefrom ON(c.conceptidfrom = valuefrom.conceptid) JOIN d_value_types dtypesfrom ON(dtypesfrom.valuetype = valuefrom.valuetype) - WHERE valueto.valuetype in (%(child_valuetypes)s) - AND valuefrom.valuetype in (%(child_valuetypes)s) + WHERE valueto.valuetype = ANY (%(child_valuetypes)s) + AND valuefrom.valuetype = ANY (%(child_valuetypes)s) ) SELECT distinct %(columns)s FROM results {offset_clause} @@ -727,10 +727,9 @@ def get_child_edges( { "conceptid": conceptid, "relationtypes": AsIs(relationtypes), - "child_valuetypes": ("','").join( - child_valuetypes + "child_valuetypes": (child_valuetypes if child_valuetypes - else models.DValueType.objects.filter(category="label").values_list("valuetype", flat=True) + else list(models.DValueType.objects.filter(category="label").values_list("valuetype", flat=True)) ), "columns": AsIs(columns), "depth_limit": depth_limit, diff --git a/arches/app/models/graph.py b/arches/app/models/graph.py index 57018ef77c1..4b1cdd13962 100644 --- a/arches/app/models/graph.py +++ b/arches/app/models/graph.py @@ -30,6 +30,7 @@ from arches.app.models.resource import Resource, UnpublishedModelError from arches.app.models.system_settings import settings from arches.app.datatypes.datatypes import DataTypeFactory +from arches.app.etl_modules.bulk_data_deletion import BulkDataDeletion from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.search.search_engine_factory import SearchEngineFactory from arches.app.utils.i18n import LanguageSynchronizer @@ -563,19 +564,18 @@ def delete(self): ) ) - def delete_instances(self, verbose=False): + def delete_instances(self, userid=None, verbose=False): """ deletes all associated resource instances """ - if verbose is True: - bar = pyprind.ProgBar(Resource.objects.filter(graph_id=self.graphid).count()) - for resource in Resource.objects.filter(graph_id=self.graphid): - resource.delete() - if verbose is True: - bar.update() - if verbose is True: - print(bar) + + bulk_deleter = BulkDataDeletion() + loadid = uuid.uuid4() + resp = bulk_deleter.delete_resources(userid, loadid, graphid=self.graphid, verbose=verbose) + bulk_deleter.index_resource_deletion(loadid) + + return resp def get_tree(self, root=None): """ diff --git a/arches/app/models/migrations/10709_refresh_geos_by_transaction.py b/arches/app/models/migrations/10709_refresh_geos_by_transaction.py index 609e94bc7f6..a6094920f4a 100644 --- a/arches/app/models/migrations/10709_refresh_geos_by_transaction.py +++ b/arches/app/models/migrations/10709_refresh_geos_by_transaction.py @@ -56,10 +56,7 @@ class Migration(migrations.Migration): $BODY$; - ALTER FUNCTION public.refresh_transaction_geojson_geometries(uuid) - OWNER TO postgres; - - + CREATE OR REPLACE FUNCTION public.__arches_staging_to_tile( load_id uuid) RETURNS boolean @@ -176,9 +173,6 @@ class Migration(migrations.Migration): RETURN status; END; $BODY$; - - ALTER FUNCTION public.__arches_staging_to_tile(uuid) - OWNER TO postgres; """ reverse = """ diff --git a/arches/app/models/migrations/10710_fix_whatisthis.py b/arches/app/models/migrations/10710_fix_whatisthis.py new file mode 100644 index 00000000000..957b691e087 --- /dev/null +++ b/arches/app/models/migrations/10710_fix_whatisthis.py @@ -0,0 +1,80 @@ +# Generated by Django 4.2.13 on 2024-05-13 09:06 + +import textwrap + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("models", "10709_refresh_geos_by_transaction"), + ] + + forward_sql = textwrap.dedent( + # Single change is uuid() in SELECT + """ + DROP VIEW IF EXISTS vw_annotations; + + CREATE OR REPLACE VIEW vw_annotations AS + SELECT uuid(json_array_elements(t.tiledata::json->n.nodeid::text->'features')->>'id') as feature_id, + t.tileid, + t.tiledata, + t.resourceinstanceid, + t.nodegroupid, + n.nodeid, + json_array_elements(t.tiledata::json->n.nodeid::text->'features')::jsonb as feature, + json_array_elements(t.tiledata::json->n.nodeid::text->'features')->'properties'->>'canvas' as canvas + FROM tiles t + LEFT JOIN nodes n ON t.nodegroupid = n.nodegroupid + WHERE ( + ( + SELECT count(*) AS count + FROM jsonb_object_keys(t.tiledata) jsonb_object_keys(jsonb_object_keys) + WHERE ( + jsonb_object_keys.jsonb_object_keys IN ( + SELECT n_1.nodeid::text AS nodeid + FROM nodes n_1 + WHERE n_1.datatype = 'annotation'::text + ) + ) + ) + ) > 0 + AND n.datatype = 'annotation'::text; + """ + ) + + reverse_sql = textwrap.dedent(""" + DROP VIEW IF EXISTS vw_annotations; + + CREATE OR REPLACE VIEW vw_annotations AS + SELECT json_array_elements(t.tiledata::json->n.nodeid::text->'features')->>'id' as feature_id, + t.tileid, + t.tiledata, + t.resourceinstanceid, + t.nodegroupid, + n.nodeid, + json_array_elements(t.tiledata::json->n.nodeid::text->'features')::jsonb as feature, + json_array_elements(t.tiledata::json->n.nodeid::text->'features')->'properties'->>'canvas' as canvas + FROM tiles t + LEFT JOIN nodes n ON t.nodegroupid = n.nodegroupid + WHERE ( + ( + SELECT count(*) AS count + FROM jsonb_object_keys(t.tiledata) jsonb_object_keys(jsonb_object_keys) + WHERE ( + jsonb_object_keys.jsonb_object_keys IN ( + SELECT n_1.nodeid::text AS nodeid + FROM nodes n_1 + WHERE n_1.datatype = 'annotation'::text + ) + ) + ) + ) > 0 + AND n.datatype = 'annotation'::text; + """ + ) + + operations = [ + migrations.RunSQL(forward_sql, reverse_sql) + ] diff --git a/arches/app/models/migrations/9191_string_nonlocalized.py b/arches/app/models/migrations/9191_string_nonlocalized.py index b5d4a0ee65b..40e87db8ca3 100644 --- a/arches/app/models/migrations/9191_string_nonlocalized.py +++ b/arches/app/models/migrations/9191_string_nonlocalized.py @@ -48,7 +48,10 @@ class Migration(migrations.Migration): "maxLength": null, "uneditable": false, "placeholder": "Enter text", - "defaultValue": "" + "defaultValue": "", + "i18n_properties": [ + "placeholder" + ] }' ) ON CONFLICT DO NOTHING; """, diff --git a/arches/app/models/models.py b/arches/app/models/models.py index 60c3bdd5d36..9cccf226619 100644 --- a/arches/app/models/models.py +++ b/arches/app/models/models.py @@ -31,7 +31,7 @@ from django.template.loader import get_template, render_to_string from django.core.validators import RegexValidator from django.db.models import Q, Max -from django.db.models.signals import post_delete, pre_save, post_save +from django.db.models.signals import post_delete, pre_save, post_save, m2m_changed from django.dispatch import receiver from django.utils import translation from django.utils.translation import gettext as _ @@ -39,8 +39,6 @@ from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType from django.core.validators import validate_slug -from guardian.models import GroupObjectPermission -from guardian.shortcuts import assign_perm # can't use "arches.app.models.system_settings.SystemSettings" because of circular refernce issue # so make sure the only settings we use in this file are ones that are static (fixed at run time) @@ -1393,19 +1391,19 @@ def is_reviewer(self): def viewable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.read_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.read_nodegroup"], any_perm=True)) @property def editable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.write_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.write_nodegroup"], any_perm=True)) @property def deletable_nodegroups(self): from arches.app.utils.permission_backend import get_nodegroups_by_perm - return set(str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(self.user, ["models.delete_nodegroup"], any_perm=True)) + return set(str(nodegroup_pk) for nodegroup_pk in get_nodegroups_by_perm(self.user, ["models.delete_nodegroup"], any_perm=True)) class Meta: managed = True @@ -1417,23 +1415,6 @@ def save_profile(sender, instance, **kwargs): UserProfile.objects.get_or_create(user=instance) -@receiver(post_save, sender=User) -def create_permissions_for_new_users(sender, instance, created, **kwargs): - from arches.app.models.resource import Resource - - if created: - ct = ContentType.objects.get(app_label="models", model="resourceinstance") - resourceInstanceIds = list(GroupObjectPermission.objects.filter(content_type=ct).values_list("object_pk", flat=True).distinct()) - for resourceInstanceId in resourceInstanceIds: - resourceInstanceId = uuid.UUID(resourceInstanceId) - resources = ResourceInstance.objects.filter(pk__in=resourceInstanceIds) - assign_perm("no_access_to_resourceinstance", instance, resources) - for resource_instance in resources: - resource = Resource(resource_instance.resourceinstanceid) - resource.graph_id = resource_instance.graph_id - resource.createdtime = resource_instance.createdtime - resource.index() - class UserXTask(models.Model): id = models.UUIDField(primary_key=True, serialize=False) @@ -1548,6 +1529,33 @@ class Meta: managed = True db_table = "user_x_notification_types" +@receiver(post_save, sender=User) +def create_permissions_for_new_users(sender, instance, created, **kwargs): + from arches.app.utils.permission_backend import process_new_user + + if created: + process_new_user(instance, created) + +@receiver(m2m_changed, sender=User.groups.through) +def update_groups_for_user(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_groups_for_user + + if action in ("post_add", "post_remove"): + update_groups_for_user(instance) + +@receiver(m2m_changed, sender=User.user_permissions.through) +def update_permissions_for_user(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_permissions_for_user + + if action in ("post_add", "post_remove"): + update_permissions_for_user(instance) + +@receiver(m2m_changed, sender=Group.permissions.through) +def update_permissions_for_group(sender, instance, action, **kwargs): + from arches.app.utils.permission_backend import update_permissions_for_group + + if action in ("post_add", "post_remove"): + update_permissions_for_group(instance) @receiver(post_save, sender=UserXNotification) def send_email_on_save(sender, instance, **kwargs): diff --git a/arches/app/models/resource.py b/arches/app/models/resource.py index b4d6d1812bd..e5448cfede3 100644 --- a/arches/app/models/resource.py +++ b/arches/app/models/resource.py @@ -40,8 +40,7 @@ from arches.app.utils import import_class_from_string, task_management from arches.app.utils.label_based_graph import LabelBasedGraph from arches.app.utils.label_based_graph_v2 import LabelBasedGraph as LabelBasedGraphV2 -from guardian.shortcuts import assign_perm, remove_perm -from guardian.exceptions import NotUserNorGroup +from arches.app.utils.permission_backend import assign_perm, remove_perm, NotUserNorGroup from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.utils.exceptions import ( InvalidNodeNameException, @@ -257,7 +256,7 @@ def load_tiles(self, user=None, perm='read_nodegroup'): self.tiles = list(models.TileModel.objects.filter(resourceinstance=self)) if user: readable_nodegroups = get_nodegroups_by_perm(user, perm, any_perm=True) - self.tiles = [tile for tile in self.tiles if tile.nodegroup is not None and tile.nodegroup in readable_nodegroups] + self.tiles = [tile for tile in self.tiles if tile.nodegroup is not None and tile.nodegroup_id in readable_nodegroups] # # flatten out the nested tiles into a single array def get_flattened_tiles(self): diff --git a/arches/app/permissions/__init__.py b/arches/app/permissions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/arches/app/permissions/arches_default_deny.py b/arches/app/permissions/arches_default_deny.py new file mode 100644 index 00000000000..3765c0094b8 --- /dev/null +++ b/arches/app/permissions/arches_default_deny.py @@ -0,0 +1,59 @@ +""" +ARCHES - a program developed to inventory and manage immovable cultural heritage. +Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from __future__ import annotations + +from arches.app.search.components.resource_type_filter import get_permitted_graphids +from django.contrib.auth.models import User +from arches.app.models.models import ResourceInstance + +from arches.app.permissions.arches_standard import ArchesStandardPermissionFramework, ResourceInstancePermissions + +class ArchesDefaultDenyPermissionFramework(ArchesStandardPermissionFramework): + def get_sets_for_user(self, user: User, perm: str) -> set[str] | None: + # We do not do set filtering - None is allow-all for sets. + return None if user and user.username != "anonymous" else set() + + + def get_restricted_users(self, resource: ResourceInstance) -> dict[str, list[int]]: + """Fetches _explicitly_ restricted users.""" + return super().get_restricted_users(resource) + + + def check_resource_instance_permissions(self, user: User, resourceid: str, permission: str) -> ResourceInstancePermissions: + result = super().check_resource_instance_permissions(user, resourceid, permission) + + if result and result.get("permitted", None) is not None: + # This is a safety check - we don't want an unpermissioned user + # defaulting to having access (allowing anonymous users is still + # possible by assigning appropriate group permissions). + if result["permitted"] == "unknown": + result["permitted"] = False + elif result["permitted"] is False: + + # This covers the case where one group denies permission and another + # allows it. Ideally, the deny would override (as normal in Arches) but + # this prevents us from having a default deny rule that another group + # can override (as deny rules in Arches must be explicit for a resource). + resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) + user_permissions = self.get_user_perms(user, resource) + if "no_access_to_resourceinstance" not in user_permissions: + group_permissions = self.get_group_perms(user, resource) + + # This should correspond to the exact case we wish to flip. + if permission in group_permissions: + result["permitted"] = True + + return result diff --git a/arches/app/permissions/arches_standard.py b/arches/app/permissions/arches_standard.py new file mode 100644 index 00000000000..d2dd968f53e --- /dev/null +++ b/arches/app/permissions/arches_standard.py @@ -0,0 +1,725 @@ +""" +ARCHES - a program developed to inventory and manage immovable cultural heritage. +Copyright (C) 2013 J. Paul Getty Trust and World Monuments Fund +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +""" + +from __future__ import annotations + +import sys +import uuid +from typing import Iterable + +from django.core.exceptions import ObjectDoesNotExist +from arches.app.models.system_settings import settings +from django.contrib.auth.models import User, Group +from django.contrib.gis.db.models import Model +from django.core.cache import caches +from guardian.backends import check_support, ObjectPermissionBackend +from guardian.core import ObjectPermissionChecker +from guardian.shortcuts import ( + get_perms, + get_group_perms, + get_user_perms, + get_users_with_perms, + get_groups_with_perms, + get_perms_for_model, +) +from guardian.exceptions import NotUserNorGroup +from arches.app.models.resource import Resource + +from guardian.models import GroupObjectPermission, UserObjectPermission, Permission +from guardian.exceptions import WrongAppError +from guardian.shortcuts import assign_perm, get_perms, remove_perm, get_group_perms, get_user_perms + +import inspect +from arches.app.models.models import * +from django.contrib.contenttypes.models import ContentType +from arches.app.models.models import ResourceInstance, MapLayer +from arches.app.search.elasticsearch_dsl_builder import Bool, Query, Terms, Nested +from arches.app.search.mappings import RESOURCES_INDEX +from arches.app.utils.permission_backend import PermissionFramework, NotUserNorGroup as ArchesNotUserNorGroup +from arches.app.search.search import SearchEngine + +if sys.version_info >= (3, 11): + from typing import NotRequired, TypedDict, Literal + class ResourceInstancePermissions(TypedDict): + permitted: NotRequired[bool | Literal["unknown"]] + resource: NotRequired[ResourceInstance] +else: + ResourceInstancePermissions = dict + +class ArchesStandardPermissionFramework(PermissionFramework): + def setup(self): + ... + + def get_perms_for_model(self, cls: str | Model) -> list[Permission]: + return get_perms_for_model(cls) # type: ignore + + def assign_perm(self, perm: Permission | str, user_or_group: User | Group, obj: ResourceInstance | None=None) -> Permission: + try: + return assign_perm(perm, user_or_group, obj=obj) + except NotUserNorGroup: + raise ArchesNotUserNorGroup() + + def get_permission_backend(self): + return PermissionBackend() + + def remove_perm(self, perm, user_or_group=None, obj=None): + return remove_perm(perm, user_or_group=user_or_group, obj=obj) + + def get_perms(self, user_or_group: User | Group, obj: ResourceInstance) -> list[Permission]: + return get_perms(user_or_group, obj) # type: ignore + + def get_group_perms(self, user_or_group: User | Group, obj: ResourceInstance) -> list[Permission]: + return get_group_perms(user_or_group, obj) # type: ignore + + def get_user_perms(self, user: User, obj: ResourceInstance) -> list[Permission]: + return get_user_perms(user, obj) # type: ignore + + def process_new_user(self, instance: User, created: bool) -> None: + ct = ContentType.objects.get(app_label="models", model="resourceinstance") + resourceInstanceIds = list(GroupObjectPermission.objects.filter(content_type=ct).values_list("object_pk", flat=True).distinct()) + for resourceInstanceId in resourceInstanceIds: + resourceInstanceId = uuid.UUID(resourceInstanceId) + resources = ResourceInstance.objects.filter(pk__in=resourceInstanceIds) + self.assign_perm("no_access_to_resourceinstance", instance, resources) + for resource_instance in resources: + resource = Resource(resource_instance.resourceinstanceid) # type: ignore + resource.graph_id = resource_instance.graph_id + resource.createdtime = resource_instance.createdtime + resource.index() # type: ignore + + def get_map_layers_by_perm(self, user: User, perms: str | Iterable[str], any_perm: bool=True) -> list[MapLayer]: + """ + returns a list of node groups that a user has the given permission on + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_map_layer" or list of strings + any_perm -- True to check ANY perm in "perms" or False to check ALL perms + + """ + + if isinstance(perms, str): + perms = [perms] + + formatted_perms = [] + # in some cases, `perms` can have a `model.` prefix + for perm in perms: + if len(perm.split(".")) > 1: + formatted_perms.append(perm.split(".")[1]) + else: + formatted_perms.append(perm) + + if user.is_superuser is True: + return list(MapLayer.objects.all()) + else: + permitted_map_layers = list() + + user_permissions = ObjectPermissionChecker(user) + + for map_layer in MapLayer.objects.all(): + if map_layer.addtomap is True and map_layer.isoverlay is False: + permitted_map_layers.append(map_layer) + else: # if no explicit permissions, object is considered accessible by all with group permissions + explicit_map_layer_perms = user_permissions.get_perms(map_layer) + if len(explicit_map_layer_perms): + if any_perm: + if len(set(formatted_perms) & set(explicit_map_layer_perms)): + permitted_map_layers.append(map_layer) + else: + if set(formatted_perms) == set(explicit_map_layer_perms): + permitted_map_layers.append(map_layer) + elif map_layer.ispublic: + permitted_map_layers.append(map_layer) + + return permitted_map_layers + + + def user_can_read_map_layers(self, user): + map_layers_with_read_permission = self.get_map_layers_by_perm(user, ["models.read_maplayer"]) + map_layers_allowed = [] + + for map_layer in map_layers_with_read_permission: + if ("no_access_to_maplayer" not in get_user_perms(user, map_layer)) or ( + map_layer.addtomap is False and map_layer.isoverlay is False + ): + map_layers_allowed.append(map_layer) + + return map_layers_allowed + + + def user_can_write_map_layers(self, user: User) -> list[MapLayer]: + map_layers_with_write_permission = self.get_map_layers_by_perm(user, ["models.write_maplayer"]) + map_layers_allowed = [] + + for map_layer in map_layers_with_write_permission: + if ("no_access_to_maplayer" not in get_user_perms(user, map_layer)) or ( + map_layer.addtomap is False and map_layer.isoverlay is False + ): + map_layers_allowed.append(map_layer) + + return map_layers_allowed + + def get_nodegroups_by_perm(self, user: User, perms: str | Iterable[str], any_perm: bool=True) -> list[str]: + """ + returns a list of node groups that a user has the given permission on + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_nodegroup" or list of strings + any_perm -- True to check ANY perm in "perms" or False to check ALL perms + + """ + return list(set( + str(nodegroup.pk) + for nodegroup in get_nodegroups_by_perm_for_user_or_group(user, perms, any_perm=any_perm) + )) + + def check_resource_instance_permissions(self, user: User, resourceid: str, permission: str) -> ResourceInstancePermissions: + """ + Checks if a user has permission to access a resource instance + + Arguments: + user -- the user to check + resourceid -- the id of the resource + permission -- the permission codename (e.g. 'view_resourceinstance') for which to check + + """ + result = ResourceInstancePermissions() + try: + if resourceid == settings.SYSTEM_SETTINGS_RESOURCE_ID: + if not user.groups.filter(name="System Administrator").exists(): + result["permitted"] = False + return result + + resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) + result["resource"] = resource + + all_perms = self.get_perms(user, resource) + + if len(all_perms) == 0: # no permissions assigned. permission implied + result["permitted"] = "unknown" + return result + else: + user_permissions = self.get_user_perms(user, resource) + if "no_access_to_resourceinstance" in user_permissions: # user is restricted + result["permitted"] = False + return result + elif permission in user_permissions: # user is permitted + result["permitted"] = True + return result + + group_permissions = self.get_group_perms(user, resource) + if "no_access_to_resourceinstance" in group_permissions: # group is restricted - no user override + result["permitted"] = False + return result + elif permission in group_permissions: # group is permitted - no user override + result["permitted"] = True + return result + + if permission not in all_perms: # neither user nor group explicitly permits or restricts. + result["permitted"] = False # restriction implied + return result + + except ObjectDoesNotExist: + result["permitted"] = True # if the object does not exist, should return true - this prevents strange 403s. + return result + + def get_users_with_perms(self, obj: Model, attach_perms: bool=False, with_superusers: bool=False, with_group_users: bool=True, only_with_perms_in: Iterable[str] | None=None) -> list[User]: + return get_users_with_perms(obj, attach_perms=attach_perms, with_superusers=with_superusers, with_group_users=with_group_users, only_with_perms_in=only_with_perms_in) # type: ignore + + def get_groups_with_perms(self, obj: Model, attach_perms: bool=False) -> list[Group]: + return get_groups_with_perms(obj, attach_perms=attach_perms) # type: ignore + + def get_restricted_users(self, resource: ResourceInstance) -> dict[str, list[int]]: + """ + Takes a resource instance and identifies which users are explicitly restricted from + reading, editing, deleting, or accessing it. + + """ + + user_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=False) + user_and_group_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=True) + + result: dict[str, list[int]] = { + "no_access": [], + "cannot_read": [], + "cannot_write": [], + "cannot_delete": [], + } + + for user, perms in user_and_group_perms.items(): + if user.is_superuser: + pass + elif user in user_perms and "no_access_to_resourceinstance" in user_perms[user]: + for k, v in result.items(): + v.append(user.id) + else: + if "view_resourceinstance" not in perms: + result["cannot_read"].append(user.id) + if "change_resourceinstance" not in perms: + result["cannot_write"].append(user.id) + if "delete_resourceinstance" not in perms: + result["cannot_delete"].append(user.id) + if "no_access_to_resourceinstance" in perms and len(perms) == 1: + result["no_access"].append(user.id) + + return result + + def get_groups_for_object(self, perm: str, obj: Model) -> list[Group]: + """ + returns a list of group objects that have the given permission on the given object + + Arguments: + perm -- the permssion string eg: "read_nodegroup" + obj -- the model instance to check + + """ + + def has_group_perm(group, perm, obj): + explicitly_defined_perms = self.get_perms(group, obj) + if len(explicitly_defined_perms) > 0: + if "no_access_to_nodegroup" in explicitly_defined_perms: + return False + else: + return perm in explicitly_defined_perms + else: + default_perms = [] + for permission in group.permissions.all(): + if perm in permission.codename: + return True + return False + + ret = [] + for group in Group.objects.all(): + if bool(has_group_perm(group, perm, obj)): # type: ignore + ret.append(group) + return ret + + + def get_sets_for_user(self, user: User, perm: str) -> set[str] | None: + # We do not do set filtering - None is allow-all for sets. + return None + + def get_users_for_object(self, perm: str, obj: Model) -> list[User]: + """ + Returns a list of user objects that have the given permission on the given object + + Arguments: + perm -- the permssion string eg: "read_nodegroup" + obj -- the model instance to check + + """ + + ret = [] + for user in User.objects.all(): + if user.has_perm(perm, obj): + ret.append(user) + return ret + + + def get_restricted_instances(self, user: User, search_engine: SearchEngine | None=None, allresources: bool=False) -> list[str]: + if allresources is False and user.is_superuser is True: + return [] + + if allresources is True: + restricted_group_instances = { + perm["object_pk"] + for perm in GroupObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") + } + restricted_user_instances = { + perm["object_pk"] + for perm in UserObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") + } + all_restricted_instances = list(restricted_group_instances | restricted_user_instances) + return all_restricted_instances + else: + terms = Terms(field="permissions.users_with_no_access", terms=[str(user.id)]) # type: ignore + query = Query(search_engine, start=0, limit=settings.SEARCH_RESULT_LIMIT) # type: ignore + has_access = Bool() # type: ignore + nested_term_filter = Nested(path="permissions", query=terms) # type: ignore + has_access.must(nested_term_filter) # type: ignore + query.add_query(has_access) # type: ignore + results = query.search(index=RESOURCES_INDEX, scroll="1m") # type: ignore + scroll_id = results["_scroll_id"] + total = results["hits"]["total"]["value"] + if total > settings.SEARCH_RESULT_LIMIT: + pages = total // settings.SEARCH_RESULT_LIMIT + for page in range(pages): + results_scrolled = query.se.es.scroll(scroll_id=scroll_id, scroll="1m") + results["hits"]["hits"] += results_scrolled["hits"]["hits"] + restricted_ids = [res["_id"] for res in results["hits"]["hits"]] + return restricted_ids + + def update_groups_for_user(self, user: User) -> None: + """Hook for spotting group updates on a user.""" + ... + + def update_permissions_for_user(self, user: User) -> None: + """Hook for spotting permission updates on a user.""" + ... + + def update_permissions_for_group(self, group: Group) -> None: + """Hook for spotting permission updates on a group.""" + ... + + def user_has_resource_model_permissions(self, user: User, perms: str | Iterable[str], resource: ResourceInstance | None=None, graph_id: str | None=None) -> bool: + """ + Checks if a user has any explicit permissions to a model's nodegroups + + Arguments: + user -- the user to check + perms -- the permssion string eg: "read_nodegroup" or list of strings + graph_id -- a graph id to check if a user has permissions to that graph's type specifically + + """ + + if resource: + graph_id = resource.graph_id + + nodegroups = self.get_nodegroups_by_perm(user, perms) + nodes = Node.objects.filter(nodegroup__in=nodegroups).filter(graph_id=graph_id).select_related("graph") + return bool(nodes.exists()) + + + def user_can_read_resource(self, user: User, resourceid: str | None=None) -> bool | None: + """ + Requires that a user be able to read an instance and read a single nodegroup of a resource + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid is not None and resourceid != "": + result = self.check_resource_instance_permissions(user, resourceid, "view_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + return self.user_has_resource_model_permissions(user, ["models.read_nodegroup"], result["resource"]) + else: + return result["permitted"] + else: + return None + + return len(self.get_resource_types_by_perm(user, ["models.read_nodegroup"])) > 0 + return False + + def get_resource_types_by_perm(self, user: User, perms: str | Iterable[str]) -> list[str]: + graphs = set() + nodegroups = self.get_nodegroups_by_perm(user, perms) + for node in Node.objects.filter(nodegroup__in=nodegroups).prefetch_related("graph"): + if node.graph.isresource and str(node.graph_id) != SystemSettings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID: + graphs.add(str(node.graph.pk)) + return list(graphs) + + + def user_can_edit_resource(self, user: User, resourceid: str | None=None) -> bool: + """ + Requires that a user be able to edit an instance and delete a single nodegroup of a resource + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid is not None and resourceid != "": + result = self.check_resource_instance_permissions(user, resourceid, "change_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or self.user_can_edit_model_nodegroups( + user, result["resource"] + ) + else: + return result["permitted"] + else: + return None + + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or len(self.get_editable_resource_types(user)) > 0 + return False + + + def user_can_delete_resource(self, user: User, resourceid: str | None=None) -> bool | None: + """ + Requires that a user be permitted to delete an instance + + """ + if user.is_authenticated: + if user.is_superuser: + return True + if resourceid is not None and resourceid != "": + result = self.check_resource_instance_permissions(user, resourceid, "delete_resourceinstance") + if result is not None: + if result["permitted"] == "unknown": + nodegroups = self.get_nodegroups_by_perm(user, "models.delete_nodegroup") + tiles = TileModel.objects.filter(resourceinstance_id=resourceid) + protected_tiles = {str(tile.nodegroup_id) for tile in tiles} - set(nodegroups) + if len(protected_tiles) > 0: + return False + return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or self.user_can_delete_model_nodegroups( + user, result["resource"] + ) + else: + return result["permitted"] + else: + return None + return False + + + def get_editable_resource_types(self, user: User) -> list[str]: + """ + returns a list of graphs of which a user can edit resource instances + + Arguments: + user -- the user to check + + """ + + if self.user_is_resource_editor(user): + return self.get_resource_types_by_perm(user, ["models.write_nodegroup", "models.delete_nodegroup"]) + else: + return [] + + + def get_createable_resource_types(self, user: User) -> list[str]: + """ + returns a list of graphs of which a user can create resource instances + + Arguments: + user -- the user to check + + """ + if self.user_is_resource_editor(user): + return self.get_resource_types_by_perm(user, "models.write_nodegroup") + else: + return [] + + + def user_can_edit_model_nodegroups(self, user: User, resource: ResourceInstance) -> bool: + """ + returns a list of graphs of which a user can edit resource instances + + Arguments: + user -- the user to check + resource -- an instance of a model + + """ + + return bool(self.user_has_resource_model_permissions(user, ["models.write_nodegroup"], resource)) + + + def user_can_delete_model_nodegroups(self, user: User, resource: ResourceInstance) -> bool: + """ + returns a list of graphs of which a user can edit resource instances + + Arguments: + user -- the user to check + resource -- an instance of a model + + """ + + return bool(self.user_has_resource_model_permissions(user, ["models.delete_nodegroup"], resource)) + + + def user_can_read_graph(self, user: User, graph_id: str) -> bool: + """ + returns a boolean denoting if a user has permmission to read a model's nodegroups + + Arguments: + user -- the user to check + graph_id -- a graph id to check if a user has permissions to that graph's type specifically + + """ + + return bool(self.user_has_resource_model_permissions(user, ["models.read_nodegroup"], graph_id=graph_id)) + + + def user_can_read_concepts(self, user: User) -> bool: + """ + Requires that a user is a part of the RDM Administrator group + + """ + + if user.is_authenticated: + return bool(user.groups.filter(name="RDM Administrator").exists()) + return False + + + def user_is_resource_editor(self, user: User) -> bool: + """ + Single test for whether a user is in the Resource Editor group + """ + + return bool(user.groups.filter(name="Resource Editor").exists()) + + + def user_is_resource_reviewer(self, user: User) -> bool: + """ + Single test for whether a user is in the Resource Reviewer group + """ + + return bool(user.groups.filter(name="Resource Reviewer").exists()) + + + def user_is_resource_exporter(self, user: User) -> bool: + """ + Single test for whether a user is in the Resource Exporter group + """ + + return bool(user.groups.filter(name="Resource Exporter").exists()) + + def user_in_group_by_name(self, user: User, names: Iterable[str]) -> bool: + return bool(user.groups.filter(name__in=names)) + + def group_required(self, user: User, *group_names: list[str]) -> bool: + # To fully reimplement this without Django groups, the following group names must (currently) be handled: + # - Application Administrator + # - RDM Administrator + # - Graph Editor + # - Resource Editor + # - Resource Exporter + # - System Administrator + + if user.is_authenticated: + if user.is_superuser or bool(user.groups.filter(name__in=group_names)): + return True + return False + + + +class PermissionBackend(ObjectPermissionBackend): # type: ignore + def has_perm(self, user_obj: User, perm: str, obj: Model | None=None) -> bool: + # check if user_obj and object are supported (pulled directly from guardian) + support, user_obj = check_support(user_obj, obj) + if not support: + return False + + if "." in perm: + app_label, perm = perm.split(".") + if obj is None: + raise RuntimeError("Passed perm has app label of '%s' and obj is None") + if app_label != obj._meta.app_label: + raise WrongAppError("Passed perm has app label of '%s' and " "given obj has '%s'" % (app_label, obj._meta.app_label)) + + obj_checker: ObjectPermissionChecker = CachedObjectPermissionChecker(user_obj, obj) + explicitly_defined_perms = obj_checker.get_perms(obj) + + if len(explicitly_defined_perms) > 0: + if "no_access_to_nodegroup" in explicitly_defined_perms: + return False + else: + return bool(perm in explicitly_defined_perms) + else: + user_checker = CachedUserPermissionChecker(user_obj) + return bool(user_checker.user_has_permission(perm)) + + +class CachedUserPermissionChecker: + """ + A permission checker that leverages the 'user_permission' cache to check user-level user permissions. + """ + + def __init__(self, user: User): + user_permission_cache = caches["user_permission"] + current_user_cached_permissions = user_permission_cache.get(str(user.pk), {}) + + if current_user_cached_permissions.get("user_permissions"): + user_permissions = current_user_cached_permissions.get("user_permissions") + else: + user_permissions = set() + + for group in user.groups.all(): + for group_permission in group.permissions.all(): + user_permissions.add(group_permission.codename) + + for user_permission in user.user_permissions.all(): + user_permissions.add(user_permission.codename) + + current_user_cached_permissions["user_permissions"] = user_permissions + user_permission_cache.set(str(user.pk), current_user_cached_permissions) + + self.user_permissions: set[str] = user_permissions + + def user_has_permission(self, permission: str) -> bool: + if permission in self.user_permissions: + return True + else: + return False + +class CachedObjectPermissionChecker: + """ + A permission checker that leverages the 'user_permission' cache to check object-level user permissions. + """ + + def __new__(cls, user: User, input: type | Model | str) -> ObjectPermissionChecker: + if inspect.isclass(input): + classname = input.__name__ + elif isinstance(input, Model): + classname = input.__class__.__name__ + elif isinstance(input, str) and globals().get(input): + classname = input + else: + raise Exception("Cannot derive model from input.") + + user_permission_cache = caches["user_permission"] + + key = f"g:{user.pk}" if isinstance(user, Group) else str(user.pk) + current_user_cached_permissions = user_permission_cache.get(key, {}) + + if current_user_cached_permissions.get(classname): + checker = current_user_cached_permissions.get(classname) + else: + checker = ObjectPermissionChecker(user) + checker.prefetch_perms(globals()[classname].objects.all()) + + current_user_cached_permissions[classname] = checker + user_permission_cache.set(key, current_user_cached_permissions) + + return checker + +def get_nodegroups_by_perm_for_user_or_group(user_or_group: User | Group, perms: str | Iterable[str] | None=None, any_perm: bool=True, ignore_perms: bool=False) -> dict[NodeGroup, set[Permission]]: + formatted_perms = [] + if perms is None: + if not ignore_perms: + raise RuntimeError("Must provide perms or explicitly ignore") + else: + if isinstance(perms, str): + perms = [perms] + + # in some cases, `perms` can have a `model.` prefix + for perm in perms: + if len(perm.split(".")) > 1: + formatted_perms.append(perm.split(".")[1]) + else: + formatted_perms.append(perm) + + permitted_nodegroups = {} + checker: ObjectPermissionChecker = CachedObjectPermissionChecker( + user_or_group, + NodeGroup, + ) + + for nodegroup in NodeGroup.objects.all(): + explicit_perms = checker.get_perms(nodegroup) + + if len(explicit_perms): + if ignore_perms: + permitted_nodegroups[nodegroup] = explicit_perms + elif any_perm: + if len(set(formatted_perms) & set(explicit_perms)): + permitted_nodegroups[nodegroup] = explicit_perms + else: + if set(formatted_perms) == set(explicit_perms): + permitted_nodegroups[nodegroup] = explicit_perms + else: # if no explicit permissions, object is considered accessible by all with group permissions + permitted_nodegroups[nodegroup] = set() + + return permitted_nodegroups diff --git a/arches/app/search/time_wheel.py b/arches/app/search/time_wheel.py index 38b174db76e..2cccf81f0df 100644 --- a/arches/app/search/time_wheel.py +++ b/arches/app/search/time_wheel.py @@ -170,7 +170,7 @@ def appendDateRanges(self, results, range_lookup): return results def get_permitted_nodegroups(self, user): - return [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(user, "models.read_nodegroup")] + return get_nodegroups_by_perm(user, "models.read_nodegroup") class d3Item(object): diff --git a/arches/app/src/declarations.d.ts b/arches/app/src/declarations.d.ts index 17dd54fb8c4..3e511106c58 100644 --- a/arches/app/src/declarations.d.ts +++ b/arches/app/src/declarations.d.ts @@ -1,72 +1,19 @@ // import declarations from other projects or Arches core -// declare modules that have been added to your project (should mirror `package.json`) +// declare untyped modules that have been added to your project in `package.json` declare module 'arches'; declare module "@babel/runtime"; -declare module "@mapbox/geojson-extent"; declare module "@mapbox/geojsonhint"; -declare module "@mapbox/mapbox-gl-draw"; -declare module "@mapbox/mapbox-gl-geocoder"; -declare module "@tmcw/togeojson"; -declare module "@turf/turf"; -declare module "backbone"; -declare module "bootstrap"; -declare module "bootstrap-colorpicker"; -declare module "chosen-js"; -declare module "ckeditor4"; -declare module "codemirror"; -declare module "core-js"; declare module "cross-env"; -declare module "cross-fetch"; -declare module "cytoscape"; declare module "cytoscape-cola"; -declare module "d3"; -declare module "datatables.net"; -declare module "datatables.net-bs"; -declare module "datatables.net-buttons"; -declare module "datatables.net-buttons-bs"; -declare module "datatables.net-responsive"; -declare module "datatables.net-responsive-bs"; -declare module "dom4"; -declare module "dropzone"; -declare module "eonasdan-bootstrap-datetimepicker"; declare module "font-awesome"; -declare module "ionicons"; -declare module "jqtree"; -declare module "jquery"; -declare module "jquery-migrate"; declare module "jquery-validation"; -declare module "jqueryui"; -declare module "js-cookie"; -declare module "knockout"; declare module "knockout-mapping"; -declare module "knockstrap"; -declare module "latlon-geohash"; -declare module "leaflet"; -declare module "leaflet-draw"; declare module "leaflet-iiif"; -declare module "leaflet.fullscreen"; declare module "lt-themify-icons"; -declare module "mapbox-gl"; -declare module "metismenu"; -declare module "moment"; -declare module "moment-timezone"; -declare module "nouislider"; -declare module "numeral"; -declare module "primevue"; -declare module "primevue/*"; -declare module "proj4"; -declare module "regenerator-runtime"; -declare module "requirejs"; declare module "requirejs-plugins"; declare module "requirejs-text"; declare module "select-woo"; -declare module "select2"; -declare module "underscore"; -declare module "uuidjs"; -declare module "vue"; -declare module "vue3-gettext"; -declare module "webpack-bundle-tracker"; // declare filetypes used in `./src/` folder declare module '*.ts'; diff --git a/arches/app/templates/javascript.htm b/arches/app/templates/javascript.htm index fedbb43da7d..53a5ae62c2b 100644 --- a/arches/app/templates/javascript.htm +++ b/arches/app/templates/javascript.htm @@ -192,6 +192,7 @@ select-a-nodegroup='{% trans "Select a nodegroup" as selectANodegroup %} "{{ selectANodegroup|escapejs }}"' no-relationships-added='{% trans "No Relationships Added" as noRelationshipsAdded %} "{{ noRelationshipsAdded|escapejs }}"' relate-resource='{% trans "Relate Resource" as relateResource %} "{{ relateResource|escapejs }}"' + configure-related-resource-relationship='{% trans "Configure Related Resource Relationship" as configureRelatedResourceRelationship %} "{{ configureRelatedResourceRelationship|escapejs }}"' cannot-be-related='{% trans "Cannot Be Related" as cannotBeRelated %} "{{ cannotBeRelated|escapejs }}"' related-resources='{% trans "Related Resources" as relatedResources %} "{{ relatedResources|escapejs }}"' view-related-resources='(resource) => {return {% trans "View resources related to ${resource}" as viewRelatedResources %} `{{ viewRelatedResources|escapejs }}` }' @@ -1048,6 +1049,7 @@ api_resources='(resourceid)=>{return "{% url "resources" "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" %}".replace("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", resourceid)}' api_search_component_data='"{% url "api_search_component_data" "aaaa"%}".replace("aaaa", "")' api_user_incomplete_workflows="{% url 'api_user_incomplete_workflows' %}" + api_get_frontend_i18n_data="{% url 'get_frontend_i18n_data' %}" geojson="{% url 'geojson' %}" icons="{% url 'icons' %}" ontology_properties="{% url 'ontology_properties' %}" diff --git a/arches/app/templates/navbar/graph-designer-menu.htm b/arches/app/templates/navbar/graph-designer-menu.htm index a130f4dd31b..ed615c5d99f 100644 --- a/arches/app/templates/navbar/graph-designer-menu.htm +++ b/arches/app/templates/navbar/graph-designer-menu.htm @@ -42,7 +42,7 @@
  • -
    +
    @@ -51,7 +51,7 @@
  • -
    +
    @@ -89,7 +89,7 @@
  • -
    +
    diff --git a/arches/app/templates/views/components/datatypes/resource-instance.htm b/arches/app/templates/views/components/datatypes/resource-instance.htm index bb7f542802f..c60232fb251 100644 --- a/arches/app/templates/views/components/datatypes/resource-instance.htm +++ b/arches/app/templates/views/components/datatypes/resource-instance.htm @@ -35,13 +35,21 @@ -
    + attr: {'aria-label': $root.translations.configureRelatedResourceRelationship, 'disabled': $parent.isEditable === false,}, + style: {color: $parent.isEditable === true || '#aaa'}, + clickBubble: false," + style="cursor: pointer;"> + + +
    +
    +
    diff --git a/arches/app/templates/views/components/widgets/resource-instance-select.htm b/arches/app/templates/views/components/widgets/resource-instance-select.htm index 8453d97a4ea..b9713078523 100644 --- a/arches/app/templates/views/components/widgets/resource-instance-select.htm +++ b/arches/app/templates/views/components/widgets/resource-instance-select.htm @@ -83,17 +83,21 @@
    - -
    + + +
    +
    - +
    @@ -321,10 +325,14 @@
    -
    + + +
    +
    diff --git a/arches/app/templates/views/rdm/modals/import-concept-form.htm b/arches/app/templates/views/rdm/modals/import-concept-form.htm index 3ace4a9231c..05acbfda7f2 100644 --- a/arches/app/templates/views/rdm/modals/import-concept-form.htm +++ b/arches/app/templates/views/rdm/modals/import-concept-form.htm @@ -18,7 +18,7 @@

    {% trans "Import Concept" %}

    - +
    @@ -177,7 +182,7 @@

    @@ -196,7 +201,7 @@

    @@ -214,7 +219,7 @@

    diff --git a/arches/app/utils/data_management/resources/formats/archesfile.py b/arches/app/utils/data_management/resources/formats/archesfile.py index e436bceee2c..42525d6ea2c 100644 --- a/arches/app/utils/data_management/resources/formats/archesfile.py +++ b/arches/app/utils/data_management/resources/formats/archesfile.py @@ -176,6 +176,7 @@ def update_or_create_tile(src_tile): "resourceinstance": resourceinstance, "parenttile_id": str(src_tile["parenttile_id"]) if src_tile["parenttile_id"] else None, "nodegroup_id": str(src_tile["nodegroup_id"]) if src_tile["nodegroup_id"] else None, + "sortorder": int(src_tile["sortorder"]) if src_tile["sortorder"] else 0, "data": src_tile["data"], } new_values = {"tileid": uuid.UUID(str(src_tile["tileid"]))} diff --git a/arches/app/utils/data_management/resources/formats/format.py b/arches/app/utils/data_management/resources/formats/format.py index 9d51b0c7a44..761048f5821 100644 --- a/arches/app/utils/data_management/resources/formats/format.py +++ b/arches/app/utils/data_management/resources/formats/format.py @@ -263,7 +263,7 @@ def get_tiles(self, graph_id=None, resourceinstanceids=None, **kwargs): user = kwargs.get("user", None) permitted_nodegroups = [] if user: - permitted_nodegroups = [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(user, "models.read_nodegroup")] + permitted_nodegroups = get_nodegroups_by_perm(user, "models.read_nodegroup") if (graph_id is None or graph_id is False) and resourceinstanceids is None: raise MissingGraphException(_("Must supply either a graph id or a list of resource instance ids to export")) diff --git a/arches/app/utils/decorators.py b/arches/app/utils/decorators.py index 5a614a56835..c387984b632 100644 --- a/arches/app/utils/decorators.py +++ b/arches/app/utils/decorators.py @@ -28,6 +28,7 @@ from arches.app.utils.permission_backend import user_can_delete_resource from arches.app.utils.permission_backend import user_can_read_concepts from arches.app.utils.permission_backend import user_created_transaction +from arches.app.utils.permission_backend import group_required as permission_group_required from django.contrib.auth.decorators import user_passes_test # Get an instance of a logger @@ -66,9 +67,7 @@ def group_required(*group_names, raise_exception=False): """ def in_groups(u): - if u.is_authenticated: - if u.is_superuser or bool(u.groups.filter(name__in=group_names)): - return True + passed = permission_group_required(u, *group_names) if raise_exception: raise PermissionDenied # else: user_passes_test() redirects to nowhere diff --git a/arches/app/utils/geo_utils.py b/arches/app/utils/geo_utils.py index 2a7ac8bfb0f..e23fa3f3f40 100644 --- a/arches/app/utils/geo_utils.py +++ b/arches/app/utils/geo_utils.py @@ -1,6 +1,7 @@ import json +import uuid from arcgis2geojson import arcgis2geojson -from django.contrib.gis.geos import GEOSGeometry, GeometryCollection +from django.contrib.gis.geos import GEOSGeometry, GeometryCollection, WKTWriter from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer @@ -67,3 +68,37 @@ def arcgisjson_to_geojson(self, geom): features.append({"type": "Feature", "properties": {}, "geometry": arcgis2geojson(geometry)}) feature_collection = {"type": "FeatureCollection", "features": features} return feature_collection + + def convert_geos_geom_collection_to_feature_collection(self, geometry): + arches_geojson = {} + arches_geojson["type"] = "FeatureCollection" + arches_geojson["features"] = [] + for geom in geometry: + arches_json_geometry = {} + arches_json_geometry["geometry"] = JSONDeserializer().deserialize(GEOSGeometry(geom, srid=4326).json) + arches_json_geometry["type"] = "Feature" + arches_json_geometry["id"] = str(uuid.uuid4()) + arches_json_geometry["properties"] = {} + arches_geojson["features"].append(arches_json_geometry) + return arches_geojson + + def reduce_precision(self, geom, current_precision): + if current_precision > 5: + writer = WKTWriter() + max_bytes = 32766 # max bytes allowed by Lucene + current_precision -= 1 + writer.precision = current_precision + less_precise_geom_collection = writer.write(GEOSGeometry(self.create_geom_collection_from_geojson(geom))) + new_byte_count = len(str(less_precise_geom_collection).encode("UTF-8")) + new_geos_geom_collection = GEOSGeometry(less_precise_geom_collection) + if new_geos_geom_collection.valid: + new_feature_collection = self.convert_geos_geom_collection_to_feature_collection(new_geos_geom_collection) + else: + raise ValueError('Geometry is not valid after reducing precision.') + if new_byte_count > max_bytes: + return self.reduce_precision(new_feature_collection, current_precision) + else: + return new_feature_collection + else: + raise ValueError('Geometry still too large after reducing precision to 5 places after the decimal.') + diff --git a/arches/app/utils/index_database.py b/arches/app/utils/index_database.py index ab68a3757a2..9653a875dd7 100644 --- a/arches/app/utils/index_database.py +++ b/arches/app/utils/index_database.py @@ -218,8 +218,7 @@ def optimize_resource_iteration(resources: Iterable[Resource], chunk_size: int): ) else: # public API that arches itself does not currently use for r in resources: - # retrieve graph -- better for this to have been selected already - r.graph + r.clean_fields() # ensure strings become UUIDs prefetch_related_objects(resources, tiles_prefetch, descriptor_prefetch) return resources diff --git a/arches/app/utils/permission_backend.py b/arches/app/utils/permission_backend.py index d6d38b43085..dea884eaf1b 100644 --- a/arches/app/utils/permission_backend.py +++ b/arches/app/utils/permission_backend.py @@ -1,621 +1,285 @@ -import inspect +from abc import abstractmethod, ABCMeta +from arches.app.const import ExtensionType from arches.app.models.models import * from arches.app.models.system_settings import settings -from guardian.backends import check_support, ObjectPermissionBackend -from django.core.exceptions import ObjectDoesNotExist -from guardian.core import ObjectPermissionChecker -from guardian.shortcuts import ( - get_perms, - get_group_perms, - get_user_perms, - get_users_with_perms, -) -from guardian.models import GroupObjectPermission, UserObjectPermission -from guardian.exceptions import WrongAppError -from django.contrib.auth.models import User, Group -from django.contrib.gis.db.models import Model -from django.core.cache import caches -from arches.app.models.models import ResourceInstance, MapLayer -from arches.app.search.elasticsearch_dsl_builder import Bool, Query, Terms, Nested -from arches.app.search.mappings import RESOURCES_INDEX - - -class PermissionBackend(ObjectPermissionBackend): - def has_perm(self, user_obj, perm, obj=None): - # check if user_obj and object are supported (pulled directly from guardian) - support, user_obj = check_support(user_obj, obj) - if not support: - return False - - if "." in perm: - app_label, perm = perm.split(".") - if app_label != obj._meta.app_label: - raise WrongAppError("Passed perm has app label of '%s' and " "given obj has '%s'" % (app_label, obj._meta.app_label)) - - ObjPermissionChecker = CachedObjectPermissionChecker(user_obj, obj) - explicitly_defined_perms = ObjPermissionChecker.get_perms(obj) - - if len(explicitly_defined_perms) > 0: - if "no_access_to_nodegroup" in explicitly_defined_perms: - return False - else: - return bool(perm in explicitly_defined_perms) - else: - UserPermissionChecker = CachedUserPermissionChecker(user_obj) - return bool(UserPermissionChecker.user_has_permission(perm)) - - -def get_restricted_users(resource): - """ - Takes a resource instance and identifies which users are explicitly restricted from - reading, editing, deleting, or accessing it. - - """ - - user_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=False) - user_and_group_perms = get_users_with_perms(resource, attach_perms=True, with_group_users=True) - - result = { - "no_access": [], - "cannot_read": [], - "cannot_write": [], - "cannot_delete": [], - } - - for user, perms in user_and_group_perms.items(): - if user.is_superuser: - pass - elif user in user_perms and "no_access_to_resourceinstance" in user_perms[user]: - for k, v in result.items(): - v.append(user.id) - else: - if "view_resourceinstance" not in perms: - result["cannot_read"].append(user.id) - if "change_resourceinstance" not in perms: - result["cannot_write"].append(user.id) - if "delete_resourceinstance" not in perms: - result["cannot_delete"].append(user.id) - if "no_access_to_resourceinstance" in perms and len(perms) == 1: - result["no_access"].append(user.id) - - return result - - -def get_restricted_instances(user, search_engine=None, allresources=False): - if allresources is False and user.is_superuser is True: - return [] - - if allresources is True: - restricted_group_instances = { - perm["object_pk"] - for perm in GroupObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") - } - restricted_user_instances = { - perm["object_pk"] - for perm in UserObjectPermission.objects.filter(permission__codename="no_access_to_resourceinstance").values("object_pk") - } - all_restricted_instances = list(restricted_group_instances | restricted_user_instances) - return all_restricted_instances - else: - terms = Terms(field="permissions.users_with_no_access", terms=[str(user.id)]) - query = Query(search_engine, start=0, limit=settings.SEARCH_RESULT_LIMIT) - has_access = Bool() - nested_term_filter = Nested(path="permissions", query=terms) - has_access.must(nested_term_filter) - query.add_query(has_access) - results = query.search(index=RESOURCES_INDEX, scroll="1m") - scroll_id = results["_scroll_id"] - total = results["hits"]["total"]["value"] - if total > settings.SEARCH_RESULT_LIMIT: - pages = total // settings.SEARCH_RESULT_LIMIT - for page in range(pages): - results_scrolled = query.se.es.scroll(scroll_id=scroll_id, scroll="1m") - results["hits"]["hits"] += results_scrolled["hits"]["hits"] - restricted_ids = [res["_id"] for res in results["hits"]["hits"]] - return restricted_ids - - -def get_groups_for_object(perm, obj): - """ - returns a list of group objects that have the given permission on the given object - - Arguments: - perm -- the permssion string eg: "read_nodegroup" - obj -- the model instance to check - - """ - - def has_group_perm(group, perm, obj): - explicitly_defined_perms = get_perms(group, obj) - if len(explicitly_defined_perms) > 0: - if "no_access_to_nodegroup" in explicitly_defined_perms: - return False - else: - return perm in explicitly_defined_perms - else: - default_perms = [] - for permission in group.permissions.all(): - if perm in permission.codename: - return True - return False - - ret = [] - for group in Group.objects.all(): - if has_group_perm(group, perm, obj): - ret.append(group) - return ret - - -def get_users_for_object(perm, obj): - """ - returns a list of user objects that have the given permission on the given object - - Arguments: - perm -- the permssion string eg: "read_nodegroup" - obj -- the model instance to check - - """ - - ret = [] - for user in User.objects.all(): - if user.has_perm(perm, obj): - ret.append(user) - return ret - - -def get_nodegroups_by_perm(user, perms, any_perm=True): - """ - returns a list of node groups that a user has the given permission on - - Arguments: - user -- the user to check - perms -- the permssion string eg: "read_nodegroup" or list of strings - any_perm -- True to check ANY perm in "perms" or False to check ALL perms - - """ - if not isinstance(perms, list): - perms = [perms] - - formatted_perms = [] - # in some cases, `perms` can have a `model.` prefix - for perm in perms: - if len(perm.split(".")) > 1: - formatted_perms.append(perm.split(".")[1]) - else: - formatted_perms.append(perm) - - permitted_nodegroups = set() - NodegroupPermissionsChecker = CachedObjectPermissionChecker(user, NodeGroup) - - for nodegroup in NodeGroup.objects.all(): - explicit_perms = NodegroupPermissionsChecker.get_perms(nodegroup) - if len(explicit_perms): - if any_perm: - if len(set(formatted_perms) & set(explicit_perms)): - permitted_nodegroups.add(nodegroup) - else: - if set(formatted_perms) == set(explicit_perms): - permitted_nodegroups.add(nodegroup) - else: # if no explicit permissions, object is considered accessible by all with group permissions - permitted_nodegroups.add(nodegroup) +class NotUserNorGroup(Exception): + ... - return permitted_nodegroups +class PermissionBackend: + def __init__(self): + self._backend = _get_permission_backend() -def get_map_layers_by_perm(user, perms, any_perm=True): - """ - returns a list of node groups that a user has the given permission on + def authenticate(self, request, username=None, password=None): + return self._backend.authenticate(request, username=username, password=password) - Arguments: - user -- the user to check - perms -- the permssion string eg: "read_map_layer" or list of strings - any_perm -- True to check ANY perm in "perms" or False to check ALL perms + def has_perm(self, user_obj, perm, obj=None): + return self._backend.has_perm(user_obj, perm, obj=obj) - """ + def get_all_permissions(self, user_obj, obj=None): + return self._backend.get_all_permissions(user_obj, obj=obj) - if not isinstance(perms, list): - perms = [perms] - formatted_perms = [] - # in some cases, `perms` can have a `model.` prefix - for perm in perms: - if len(perm.split(".")) > 1: - formatted_perms.append(perm.split(".")[1]) +def user_created_transaction(user, transactionid): + if user.is_authenticated: + if user.is_superuser: + return True + if EditLog.objects.filter(transactionid=transactionid).exists(): + if EditLog.objects.filter(transactionid=transactionid, userid=user.id).exists(): + return True else: - formatted_perms.append(perm) - - if user.is_superuser is True: - return MapLayer.objects.all() - else: - permitted_map_layers = list() - - user_permissions = ObjectPermissionChecker(user) - - for map_layer in MapLayer.objects.all(): - if map_layer.addtomap is True and map_layer.isoverlay is False: - permitted_map_layers.append(map_layer) - else: # if no explicit permissions, object is considered accessible by all with group permissions - explicit_map_layer_perms = user_permissions.get_perms(map_layer) - if len(explicit_map_layer_perms): - if any_perm: - if len(set(formatted_perms) & set(explicit_map_layer_perms)): - permitted_map_layers.append(map_layer) - else: - if set(formatted_perms) == set(explicit_map_layer_perms): - permitted_map_layers.append(map_layer) - elif map_layer.ispublic: - permitted_map_layers.append(map_layer) - - return permitted_map_layers - + return True + return False -def user_can_read_map_layers(user): - map_layers_with_read_permission = get_map_layers_by_perm(user, ["models.read_maplayer"]) - map_layers_allowed = [] +class PermissionFramework(metaclass=ABCMeta): + @abstractmethod + def user_can_read_graph(self, user, graph_id): + ... - for map_layer in map_layers_with_read_permission: - if ("no_access_to_maplayer" not in get_user_perms(user, map_layer)) or ( - map_layer.addtomap is False and map_layer.isoverlay is False - ): - map_layers_allowed.append(map_layer) + @abstractmethod + def user_can_delete_model_nodegroups(self, user, resource): + ... - return map_layers_allowed + @abstractmethod + def user_can_edit_model_nodegroups(self, user, resource): + ... + @abstractmethod + def get_createable_resource_types(self, user): + ... -def user_can_write_map_layers(user): - map_layers_with_write_permission = get_map_layers_by_perm(user, ["models.write_maplayer"]) - map_layers_allowed = [] + @abstractmethod + def get_editable_resource_types(self, user): + ... - for map_layer in map_layers_with_write_permission: - if ("no_access_to_maplayer" not in get_user_perms(user, map_layer)) or ( - map_layer.addtomap is False and map_layer.isoverlay is False - ): - map_layers_allowed.append(map_layer) + @abstractmethod + def assign_perm(self, perm, user_or_group, obj=None): + ... - return map_layers_allowed + @abstractmethod + def remove_perm(self, perm, user_or_group=None, obj=None): + ... + @abstractmethod + def get_permission_backend(self): + ... -def get_editable_resource_types(user): - """ - returns a list of graphs of which a user can edit resource instances + @abstractmethod + def get_restricted_users(self, resource): + ... - Arguments: - user -- the user to check + @abstractmethod + def get_restricted_instances(self, user, search_engine=None, allresources=False): + ... - """ + @abstractmethod + def get_groups_for_object(self, perm, obj): + ... - if user_is_resource_editor(user): - return get_resource_types_by_perm(user, ["models.write_nodegroup", "models.delete_nodegroup"]) - else: - return [] + @abstractmethod + def get_users_for_object(self, perm, obj): + ... + @abstractmethod + def check_resource_instance_permissions(self, user, resourceid, permission): + ... -def get_createable_resource_types(user): - """ - returns a list of graphs of which a user can create resource instances + @abstractmethod + def get_nodegroups_by_perm(self, user, perms, any_perm=True): + ... - Arguments: - user -- the user to check + @abstractmethod + def get_map_layers_by_perm(self, user, perms, any_perm=True): + ... - """ - if user_is_resource_editor(user): - return get_resource_types_by_perm(user, "models.write_nodegroup") - else: - return [] + @abstractmethod + def user_can_read_map_layers(self, user): + ... + @abstractmethod + def user_can_write_map_layers(self, user): + ... -def get_resource_types_by_perm(user, perms): - """ - returns a list of graphs for which a user has specific nodegroup permissions + @abstractmethod + def process_new_user(self, instance, created): + ... - Arguments: - user -- the user to check - perms -- the permssion string eg: "read_nodegroup" or list of strings - resource -- a resource instance to check if a user has permissions to that resource's type specifically + @abstractmethod + def user_has_resource_model_permissions(self, user, perms, resource): + ... - """ + @abstractmethod + def user_can_read_resource(self, user, resourceid=None): + ... - graphs = set() - nodegroups = get_nodegroups_by_perm(user, perms) - for node in Node.objects.filter(nodegroup__in=nodegroups).prefetch_related("graph"): - if node.graph.isresource and str(node.graph_id) != settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID: - graphs.add(node.graph) - return list(graphs) + @abstractmethod + def user_can_edit_resource(self, user, resourceid=None): + ... + @abstractmethod + def user_can_delete_resource(self, user, resourceid=None): + ... -def user_can_edit_model_nodegroups(user, resource): - """ - returns a list of graphs of which a user can edit resource instances + @abstractmethod + def user_can_read_concepts(self, user): + ... - Arguments: - user -- the user to check - resource -- an instance of a model + @abstractmethod + def user_is_resource_editor(self, user): + ... - """ + @abstractmethod + def user_is_resource_reviewer(self, user): + ... + + @abstractmethod + def user_is_resource_exporter(self, user): + ... + + @abstractmethod + def get_resource_types_by_perm(self, user, perms): + ... + + @abstractmethod + def user_in_group_by_name(self, user, names): + ... + + @abstractmethod + def group_required(self, user, *group_names): + ... + +_PERMISSION_FRAMEWORK = None + +def _get_permission_framework(): + global _PERMISSION_FRAMEWORK + if not _PERMISSION_FRAMEWORK: + if settings.PERMISSION_FRAMEWORK: + if "." not in settings.PERMISSION_FRAMEWORK: + raise RuntimeError("Permissions frameworks must be a dot-separated module and a class") + modulename, classname = settings.PERMISSION_FRAMEWORK.split(".", -1) + PermissionFramework = get_class_from_modulename(modulename, classname, ExtensionType.PERMISSIONS_FRAMEWORKS) + _PERMISSION_FRAMEWORK = PermissionFramework() + else: + from arches.app.permissions.arches_standard import ArchesStandardPermissionFramework + _PERMISSION_FRAMEWORK = ArchesStandardPermissionFramework() + return _PERMISSION_FRAMEWORK - return user_has_resource_model_permissions(user, ["models.write_nodegroup"], resource) +def get_createable_resource_models(user): + return GraphModel.objects.filter(pk__in=list(get_createable_resource_types(user))).all() +def assign_perm(perm, user_or_group, obj=None): + return _get_permission_framework().assign_perm(perm, user_or_group, obj=obj) -def user_can_delete_model_nodegroups(user, resource): - """ - returns a list of graphs of which a user can edit resource instances +def remove_perm(perm, user_or_group=None, obj=None): + return _get_permission_framework().remove_perm(perm, user_or_group=user_or_group, obj=obj) - Arguments: - user -- the user to check - resource -- an instance of a model +def _get_permission_backend(): + return _get_permission_framework().get_permission_backend() - """ +def get_restricted_users(resource): + return _get_permission_framework().get_restricted_users(resource) - return user_has_resource_model_permissions(user, ["models.delete_nodegroup"], resource) +def get_restricted_instances(user, search_engine=None, allresources=False): + return _get_permission_framework().get_restricted_instances(user, search_engine=search_engine, allresources=allresources) +def get_groups_for_object(perm, obj): + return _get_permission_framework().get_groups_for_object(perm, obj) -def user_can_read_graph(user, graph_id): - """ - returns a boolean denoting if a user has permmission to read a model's nodegroups +def get_users_for_object(perm, obj): + return _get_permission_framework().get_users_for_object(perm, obj) - Arguments: - user -- the user to check - graph_id -- a graph id to check if a user has permissions to that graph's type specifically +def check_resource_instance_permissions(user, resourceid, permission): + return _get_permission_framework().check_resource_instance_permissions(user, resourceid, permission) - """ +def get_nodegroups_by_perm(user, perms, any_perm=True): + return _get_permission_framework().get_nodegroups_by_perm(user, perms, any_perm=any_perm) - return user_has_resource_model_permissions(user, ["models.read_nodegroup"], graph_id=graph_id) +def get_map_layers_by_perm(user, perms, any_perm=True): + return _get_permission_framework().get_map_layers_by_perm(user, perms, any_perm=any_perm) +def user_can_read_map_layers(user): + return _get_permission_framework().user_can_read_map_layers(user) -def user_has_resource_model_permissions(user, perms, resource=None, graph_id=None): - """ - Checks if a user has any explicit permissions to a model's nodegroups +def user_can_write_map_layers(user): + return _get_permission_framework().user_can_write_map_layers(user) - Arguments: - user -- the user to check - perms -- the permssion string eg: "read_nodegroup" or list of strings - graph_id -- a graph id to check if a user has permissions to that graph's type specifically +def get_users_with_perms(obj, attach_perms=False, with_superusers=False, with_group_users=True, only_with_perms_in=None): + return _get_permission_framework().get_users_with_perms(obj, attach_perms=attach_perms, with_superusers=with_superusers, with_group_users=with_group_users, only_with_perms_in=only_with_perms_in) - """ +def get_groups_with_perms(obj, attach_perms=False): + return _get_permission_framework().get_groups_with_perms(obj, attach_perms=attach_perms) - if resource: - graph_id = resource.graph_id +def get_user_perms(user, obj): + return _get_permission_framework().get_user_perms(user, obj) - nodegroups = get_nodegroups_by_perm(user, perms) - nodes = Node.objects.filter(nodegroup__in=nodegroups).filter(graph_id=graph_id).select_related("graph") - return nodes.exists() +def get_group_perms(user_or_group, obj): + return _get_permission_framework().get_group_perms(user_or_group, obj) +def get_perms_for_model(cls): + return _get_permission_framework().get_perms_for_model(cls) -def check_resource_instance_permissions(user, resourceid, permission): - """ - Checks if a user has permission to access a resource instance - - Arguments: - user -- the user to check - resourceid -- the id of the resource - permission -- the permission codename (e.g. 'view_resourceinstance') for which to check - - """ - result = {} - try: - if resourceid == settings.SYSTEM_SETTINGS_RESOURCE_ID: - if not user.groups.filter(name="System Administrator").exists(): - result["permitted"] = False - return result - - resource = ResourceInstance.objects.get(resourceinstanceid=resourceid) - result["resource"] = resource - - all_perms = get_perms(user, resource) - - if len(all_perms) == 0: # no permissions assigned. permission implied - result["permitted"] = "unknown" - return result - else: - user_permissions = get_user_perms(user, resource) - if "no_access_to_resourceinstance" in user_permissions: # user is restricted - result["permitted"] = False - return result - elif permission in user_permissions: # user is permitted - result["permitted"] = True - return result +def get_perms(user_or_group, obj): + return _get_permission_framework().get_perms(user_or_group, obj) - group_permissions = get_group_perms(user, resource) - if "no_access_to_resourceinstance" in group_permissions: # group is restricted - no user override - result["permitted"] = False - return result - elif permission in group_permissions: # group is permitted - no user override - result["permitted"] = True - return result +def process_new_user(instance, created): + return _get_permission_framework().process_new_user(instance, created) - if permission not in all_perms: # neither user nor group explicitly permits or restricts. - result["permitted"] = False # restriction implied - return result +def update_groups_for_user(instance): + return _get_permission_framework().update_groups_for_user(instance) - except ObjectDoesNotExist: - result["permitted"] = True # if the object does not exist, no harm in returning true - this prevents strange 403s. - return result +def update_permissions_for_user(instance): + return _get_permission_framework().update_permissions_for_user(instance) - return result +def update_permissions_for_group(instance): + return _get_permission_framework().update_permissions_for_group(instance) +def user_has_resource_model_permissions(user, perms, resource): + return _get_permission_framework().user_has_resource_model_permissions(user, perms, resource) def user_can_read_resource(user, resourceid=None): - """ - Requires that a user be able to read an instance and read a single nodegroup of a resource - - """ - if user.is_authenticated: - if user.is_superuser: - return True - if resourceid not in [None, ""]: - result = check_resource_instance_permissions(user, resourceid, "view_resourceinstance") - if result is not None: - if result["permitted"] == "unknown": - return user_has_resource_model_permissions(user, ["models.read_nodegroup"], result["resource"]) - else: - return result["permitted"] - else: - return None - - return len(get_resource_types_by_perm(user, ["models.read_nodegroup"])) > 0 - return False - + return _get_permission_framework().user_can_read_resource(user, resourceid=resourceid) def user_can_edit_resource(user, resourceid=None): - """ - Requires that a user be able to edit an instance and delete a single nodegroup of a resource - - """ - if user.is_authenticated: - if user.is_superuser: - return True - if resourceid not in [None, ""]: - result = check_resource_instance_permissions(user, resourceid, "change_resourceinstance") - if result is not None: - if result["permitted"] == "unknown": - return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or user_can_edit_model_nodegroups( - user, result["resource"] - ) - else: - return result["permitted"] - else: - return None - - return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or len(get_editable_resource_types(user)) > 0 - return False - + return _get_permission_framework().user_can_edit_resource(user, resourceid=resourceid) def user_can_delete_resource(user, resourceid=None): - """ - Requires that a user be permitted to delete an instance - - """ - if user.is_authenticated: - if user.is_superuser: - return True - if resourceid not in [None, ""]: - result = check_resource_instance_permissions(user, resourceid, "delete_resourceinstance") - if result is not None: - if result["permitted"] == "unknown": - nodegroups = get_nodegroups_by_perm(user, "models.delete_nodegroup") - tiles = TileModel.objects.filter(resourceinstance_id=resourceid) - protected_tiles = {str(tile.nodegroup_id) for tile in tiles} - {str(nodegroup.nodegroupid) for nodegroup in nodegroups} - if len(protected_tiles) > 0: - return False - return user.groups.filter(name__in=settings.RESOURCE_EDITOR_GROUPS).exists() or user_can_delete_model_nodegroups( - user, result["resource"] - ) - else: - return result["permitted"] - else: - return None - return False - + return _get_permission_framework().user_can_delete_resource(user, resourceid=resourceid) def user_can_read_concepts(user): - """ - Requires that a user is a part of the RDM Administrator group - - """ - - if user.is_authenticated: - return user.groups.filter(name="RDM Administrator").exists() - return False - + return _get_permission_framework().user_can_read_concepts(user) def user_is_resource_editor(user): - """ - Single test for whether a user is in the Resource Editor group - """ - - return user.groups.filter(name="Resource Editor").exists() - + return _get_permission_framework().user_is_resource_editor(user) def user_is_resource_reviewer(user): - """ - Single test for whether a user is in the Resource Reviewer group - """ - - return user.groups.filter(name="Resource Reviewer").exists() - + return _get_permission_framework().user_is_resource_reviewer(user) def user_is_resource_exporter(user): - """ - Single test for whether a user is in the Resource Exporter group - """ - - return user.groups.filter(name="Resource Exporter").exists() + return _get_permission_framework().user_is_resource_exporter(user) +def get_resource_types_by_perm(user, perms): + return _get_permission_framework().get_resource_types_by_perm(user, perms) -def user_created_transaction(user, transactionid): - if user.is_authenticated: - if user.is_superuser: - return True - if EditLog.objects.filter(transactionid=transactionid).exists(): - if EditLog.objects.filter(transactionid=transactionid, userid=user.id).exists(): - return True - else: - return True - return False - - -class CachedObjectPermissionChecker: - """ - A permission checker that leverages the 'user_permission' cache to check object-level user permissions. - """ - - def __new__(cls, user, input): - if inspect.isclass(input): - classname = input.__name__ - elif isinstance(input, Model): - classname = input.__class__.__name__ - elif isinstance(input, str) and globals().get(input): - classname = input - else: - raise Exception("Cannot derive model from input.") - - user_permission_cache = caches["user_permission"] - - current_user_cached_permissions = user_permission_cache.get(str(user.pk), {}) - - if current_user_cached_permissions.get(classname): - checker = current_user_cached_permissions.get(classname) - else: - checker = ObjectPermissionChecker(user) - checker.prefetch_perms(globals()[classname].objects.all()) - - current_user_cached_permissions[classname] = checker - user_permission_cache.set(str(user.pk), current_user_cached_permissions) - - return checker - - -class CachedUserPermissionChecker: - """ - A permission checker that leverages the 'user_permission' cache to check user-level user permissions. - """ - - def __init__(self, user): - user_permission_cache = caches["user_permission"] - current_user_cached_permissions = user_permission_cache.get(str(user.pk), {}) +def user_in_group_by_name(user, names): + return _get_permission_framework().user_in_group_by_name(user, names) - if current_user_cached_permissions.get("user_permissions"): - user_permissions = current_user_cached_permissions.get("user_permissions") - else: - user_permissions = set() +def user_can_read_graph(user, graph_id): + return _get_permission_framework().user_can_read_graph(user, graph_id) - for group in user.groups.all(): - for group_permission in group.permissions.all(): - user_permissions.add(group_permission.codename) +def user_can_delete_model_nodegroups(user, resource): + return _get_permission_framework().user_can_delete_model_nodegroups(user, resource) - for user_permission in user.user_permissions.all(): - user_permissions.add(user_permission.codename) +def user_can_edit_model_nodegroups(user, resource): + return _get_permission_framework().user_can_edit_model_nodegroups(user, resource) - current_user_cached_permissions["user_permissions"] = user_permissions - user_permission_cache.set(str(user.pk), current_user_cached_permissions) +def get_createable_resource_types(user): + return _get_permission_framework().get_createable_resource_types(user) - self.user_permissions = user_permissions +def get_editable_resource_types(user): + return _get_permission_framework().get_editable_resource_types(user) - def user_has_permission(self, permission): - if permission in self.user_permissions: - return True - else: - return False +def group_required(user, *group_names): + return _get_permission_framework().group_required(user, *group_names) diff --git a/arches/app/utils/string_utils.py b/arches/app/utils/string_utils.py index e6596c4b887..8ce5ec86b62 100644 --- a/arches/app/utils/string_utils.py +++ b/arches/app/utils/string_utils.py @@ -1,5 +1,5 @@ def str_to_bool(value): - match value: + match value.lower(): case "y" | "yes" | "t" | "true" | "on" | "1": return True case "n" | "no" | "f" | "false" | "off" | "0": diff --git a/arches/app/views/api.py b/arches/app/views/api.py index 1c2b4ef6f34..429c19d3dbb 100644 --- a/arches/app/views/api.py +++ b/arches/app/views/api.py @@ -542,12 +542,19 @@ def get(self, request, resourceid=None, slug=None, graphid=None): elif format == "json-ld": try: - models.ResourceInstance.objects.get(pk=resourceid) # check for existance + resource = models.ResourceInstance.objects.select_related("graph").get(pk=resourceid) + if not resource.graph.ontology_id: + return JSONErrorResponse( + message=_( + "The graph '{0}' does not have an ontology. JSON-LD requires one." + ).format(resource.graph.name), + status=400, + ) exporter = ResourceExporter(format=format) output = exporter.writer.write_resources(resourceinstanceids=[resourceid], indent=indent, user=request.user) out = output[0]["outputfile"].getvalue() except models.ResourceInstance.DoesNotExist: - logger.error(_("The specified resource '{0}' does not exist. JSON-LD export failed.".format(resourceid))) + logger.error(_("The specified resource '{0}' does not exist. JSON-LD export failed.").format(resourceid)) return JSONResponse(status=404) else: @@ -1044,7 +1051,7 @@ def get(self, request): canvas = request.GET.get("canvas", None) resourceid = request.GET.get("resourceid", None) nodeid = request.GET.get("nodeid", None) - permitted_nodegroups = [nodegroup for nodegroup in get_nodegroups_by_perm(request.user, "models.read_nodegroup")] + permitted_nodegroups = get_nodegroups_by_perm(request.user, "models.read_nodegroup") annotations = models.VwAnnotation.objects.filter(nodegroup__in=permitted_nodegroups) if canvas is not None: annotations = annotations.filter(canvas=canvas) @@ -1079,7 +1086,7 @@ def get(self, request): class IIIFAnnotationNodes(APIBase): def get(self, request, indent=None): - permitted_nodegroups = [nodegroup for nodegroup in get_nodegroups_by_perm(request.user, "models.read_nodegroup")] + permitted_nodegroups = get_nodegroups_by_perm(request.user, "models.read_nodegroup") annotation_nodes = models.Node.objects.filter(nodegroup__in=permitted_nodegroups, datatype="annotation") return JSONResponse( [ @@ -1366,7 +1373,7 @@ def get(self, request, tileid): return JSONResponse(str(e), status=404) # filter tiles from attribute query based on user permissions - permitted_nodegroups = [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(request.user, "models.read_nodegroup")] + permitted_nodegroups = get_nodegroups_by_perm(request.user, "models.read_nodegroup") if str(tile.nodegroup_id) in permitted_nodegroups: return JSONResponse(tile, status=200) else: @@ -1397,7 +1404,7 @@ def get(self, request, nodegroupid=None): try: nodegroup = models.NodeGroup.objects.get(pk=params["nodegroupid"]) - permitted_nodegroups = [nodegroup.pk for nodegroup in get_nodegroups_by_perm(user, perms)] + permitted_nodegroups = get_nodegroups_by_perm(user, perms) except Exception as e: return JSONResponse(str(e), status=404) @@ -1445,7 +1452,7 @@ def graphLookup(graphid): # try to get nodes by attribute filter and then get nodes by passed in user perms try: nodes = models.Node.objects.filter(**dict(params)).values() - permitted_nodegroups = [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(user, perms)] + permitted_nodegroups = get_nodegroups_by_perm(user, perms) except Exception as e: return JSONResponse(str(e), status=404) diff --git a/arches/app/views/base.py b/arches/app/views/base.py index 7b2d799f80e..49813cbd338 100644 --- a/arches/app/views/base.py +++ b/arches/app/views/base.py @@ -26,13 +26,12 @@ from django.views.generic import TemplateView from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.utils.permission_backend import ( - get_createable_resource_types, + get_createable_resource_models, user_is_resource_reviewer, get_editable_resource_types, get_resource_types_by_perm, user_can_read_map_layers, ) -from arches.app.utils.permission_backend import get_createable_resource_types, user_is_resource_reviewer class BaseManagerView(TemplateView): @@ -52,7 +51,7 @@ def get_context_data(self, **kwargs): if self.request.user.has_perm("view_plugin", plugin): context["plugins"].append(plugin) - createable = get_createable_resource_types(self.request.user) + createable = list(get_createable_resource_models(self.request.user)) createable.sort(key=lambda x: x.name.lower()) context["createable_resources"] = JSONSerializer().serialize( createable, diff --git a/arches/app/views/graph.py b/arches/app/views/graph.py index 74eb9d4b962..5c1e6433dc6 100644 --- a/arches/app/views/graph.py +++ b/arches/app/views/graph.py @@ -45,7 +45,7 @@ from arches.app.utils.data_management.resource_graphs import importer as GraphImporter from arches.app.utils.system_metadata import system_metadata from arches.app.views.base import BaseManagerView -from guardian.shortcuts import assign_perm, get_perms, remove_perm, get_group_perms, get_user_perms +from arches.app.utils.permission_backend import assign_perm, get_perms, remove_perm, get_group_perms, get_user_perms from io import BytesIO from elasticsearch.exceptions import RequestError from django.core.cache import cache @@ -189,7 +189,7 @@ def get_ontology_namespaces(self): def get(self, request, graphid): if graphid == settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID: - if not request.user.groups.filter(name="System Administrator").exists(): + if not group_required("System Administrator", raise_exception=True): raise PermissionDenied self.graph = Graph.objects.get(graphid=graphid) @@ -422,13 +422,20 @@ def post(self, request, graphid=None): clone_data = graph.copy(root=data) clone_data["copy"].slug = None clone_data["copy"].save() + ret = {"success": True, "graphid": clone_data["copy"].pk} elif self.action == "clone_graph": clone_data = graph.copy() ret = clone_data["copy"] ret.slug = None + ret.publication = None + ret.save() + + if bool(graph.publication_id): + ret.publish(user=request.user) + ret.copy_functions(graph, [clone_data["nodes"], clone_data["nodegroups"]]) elif self.action == "reorder_nodes": @@ -469,12 +476,13 @@ def delete(self, request, graphid): elif self.action == "delete_instances": try: graph = Graph.objects.get(graphid=graphid) - graph.delete_instances() + resp = graph.delete_instances(userid=request.user.id) + success = resp["success"] return JSONResponse( { - "success": True, - "message": "All the resources associated with the Model '{0}' have been successfully deleted.".format(graph.name), - "title": "Resources Successfully Deleted.", + "success": resp["success"], + "message": resp["message"], + "title": f"Resources {'Successfully' if success else 'Unsuccessfully'} Deleted from {graph.name}.", } ) except GraphValidationError as e: @@ -485,7 +493,7 @@ def delete(self, request, graphid): try: graph = Graph.objects.get(graphid=graphid) if graph.isresource: - graph.delete_instances() + graph.delete_instances(userid=request.user.id) graph.delete() return JSONResponse({"success": True}) except GraphValidationError as e: diff --git a/arches/app/views/map.py b/arches/app/views/map.py index 3eca63c7654..0e7d9854fc8 100644 --- a/arches/app/views/map.py +++ b/arches/app/views/map.py @@ -20,7 +20,6 @@ from django.http import Http404 from django.utils.translation import gettext as _ from django.utils.decorators import method_decorator -from guardian.shortcuts import get_users_with_perms, get_groups_with_perms from revproxy.views import ProxyView from arches.app.models import models from arches.app.models.system_settings import settings @@ -30,7 +29,7 @@ from arches.app.utils.betterJSONSerializer import JSONSerializer, JSONDeserializer from arches.app.utils.decorators import group_required from arches.app.utils.response import JSONResponse -from arches.app.utils.permission_backend import get_users_for_object, get_groups_for_object +from arches.app.utils.permission_backend import get_users_for_object, get_groups_for_object, get_users_with_perms, get_groups_with_perms from arches.app.search.search_engine_factory import SearchEngineFactory from arches.app.search.elasticsearch_dsl_builder import Query, Bool, GeoBoundsAgg, Term from arches.app.search.mappings import RESOURCES_INDEX diff --git a/arches/app/views/resource.py b/arches/app/views/resource.py index ea266274dc0..55de3346d1e 100644 --- a/arches/app/views/resource.py +++ b/arches/app/views/resource.py @@ -65,7 +65,7 @@ from arches.app.views.concept import Concept from arches.app.datatypes.datatypes import DataTypeFactory from elasticsearch import Elasticsearch -from guardian.shortcuts import ( +from arches.app.utils.permission_backend import ( assign_perm, get_perms, remove_perm, diff --git a/arches/app/views/search.py b/arches/app/views/search.py index b229430d50a..570123641c6 100644 --- a/arches/app/views/search.py +++ b/arches/app/views/search.py @@ -443,7 +443,7 @@ def get_provisional_type(request): def get_permitted_nodegroups(user): - return [str(nodegroup.pk) for nodegroup in get_nodegroups_by_perm(user, "models.read_nodegroup")] + return get_nodegroups_by_perm(user, "models.read_nodegroup") def buffer(request): diff --git a/arches/install/arches-admin b/arches/install/arches-admin index 41373c87f4d..96de9431ca9 100644 --- a/arches/install/arches-admin +++ b/arches/install/arches-admin @@ -116,7 +116,7 @@ class ArchesProjectCommand(TemplateCommand): # need to manually replace instances of {{ project_name }} in some files path_to_project = os.path.join(target) if target else os.path.join(os.getcwd(), project_name) - for relative_file_path in [".yarnrc", "pyproject.toml"]: # relative to app root directory + for relative_file_path in ['.coveragerc', "pyproject.toml"]: # relative to app root directory file = open(os.path.join(path_to_project, relative_file_path),'r') file_data = file.read() file.close() diff --git a/arches/install/arches-project b/arches/install/arches-project index fb9734fddbc..b6fe7896374 100644 --- a/arches/install/arches-project +++ b/arches/install/arches-project @@ -53,7 +53,7 @@ class ArchesCommand(TemplateCommand): # need to manually replace instances of {{ project_name }} in some files path_to_project = os.path.join(target) if target else os.path.join(os.getcwd(), project_name) - for relative_file_path in [".yarnrc", "pyproject.toml"]: # relative to app root directory + for relative_file_path in ['.coveragerc', "pyproject.toml"]: # relative to app root directory file = open(os.path.join(path_to_project, relative_file_path),'r') file_data = file.read() file.close() diff --git a/arches/install/arches-templates/.coveragerc b/arches/install/arches-templates/.coveragerc new file mode 100644 index 00000000000..ffa4a27fbf8 --- /dev/null +++ b/arches/install/arches-templates/.coveragerc @@ -0,0 +1,29 @@ +[run] +source = + {{ project_name }}/ + +omit = + */python?.?/* + */models/migrations/* + */settings*.py + */urls.py + */wsgi.py + */celery.py + */__init__.py + +data_file = coverage/python/.coverage + +[report] +show_missing = true + +exclude_lines = + pragma: no cover + +[html] +directory = coverage/python/htmlcov + +[xml] +output = coverage/python/coverage.xml + +[json] +output = coverage/python/coverage.json diff --git a/arches/install/arches-templates/.eslintignore b/arches/install/arches-templates/.eslintignore deleted file mode 100644 index 00d347c90f6..00000000000 --- a/arches/install/arches-templates/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -!.eslintrc.js \ No newline at end of file diff --git a/arches/install/arches-templates/.eslintrc.js b/arches/install/arches-templates/.eslintrc.js deleted file mode 100644 index 33b18321c59..00000000000 --- a/arches/install/arches-templates/.eslintrc.js +++ /dev/null @@ -1,63 +0,0 @@ -module.exports = { - "extends": [ - "eslint:recommended", - 'plugin:@typescript-eslint/recommended', - 'plugin:vue/vue3-recommended', - ], - "root": true, - "env": { - "browser": true, - "es6": true, - "node": true - }, - "parser": "vue-eslint-parser", - "parserOptions": { - "ecmaVersion": 11, - "sourceType": "module", - "requireConfigFile": false, - "parser": { - "ts": "@typescript-eslint/parser" - } - }, - "globals": { - "define": false, - "require": false, - "window": false, - "console": false, - "history": false, - "location": false, - "Promise": false, - "setTimeout": false, - "URL": false, - "URLSearchParams": false, - "fetch": false - }, - "rules": { - "semi": ["error", "always"], - }, - "overrides": [ - { - "files": [ "*.vue" ], - "rules": { - "vue/html-indent": [2, 4], - } - }, - { - "files": [ "*.js" ], - "rules": { - "indent": ["error", 4], - "space-before-function-paren": ["error", "never"], - "no-extra-boolean-cast": 0, // 0=silence, 1=warning, 2=error - // allow async-await - 'generator-star-spacing': 'off', - // allow debugger during development - 'no-debugger': process.env.NODE_ENV === 'production' ? 'error' : 'off', - 'no-unused-vars': [1, { - argsIgnorePattern: '^_' - }], - "camelcase": [1, {"properties": "always"}], - } - } - ] -}; - \ No newline at end of file diff --git a/arches/install/arches-templates/.github/workflows/main.yml b/arches/install/arches-templates/.github/workflows/main.yml new file mode 100644 index 00000000000..aa77801b40c --- /dev/null +++ b/arches/install/arches-templates/.github/workflows/main.yml @@ -0,0 +1,404 @@ +name: CI + +on: + # push: -- just run on PRs for now + pull_request: + workflow_dispatch: + +jobs: + build_feature_branch: + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: ${{ github.event.repository.name }} + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + + - name: Install Java, GDAL, and other system dependencies + run: | + sudo apt update + sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev libxslt-dev + echo Postgres and ES dependencies installed + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install . + pip install -r ${{ github.event.repository.name }}/install/requirements.txt + pip install -r ${{ github.event.repository.name }}/install/requirements_dev.txt + echo Python packages installed + + - uses: ankane/setup-elasticsearch@v1 + with: + elasticsearch-version: 8 + + - name: Webpack frontend files + run: | + echo "Checking for yarn.lock file..." + if [ -f yarn.lock ]; then + echo "Removing yarn.lock due to yarn v1 package resolution issues" + echo "https://github.com/iarna/wide-align/issues/63" + rm yarn.lock + else + echo "yarn.lock not found, skipping remove." + fi + + echo "Checking for package.json..." + if [ -f package.json ]; then + echo "package.json found, building static bundle." + yarn && yarn build_test + else + echo "package.json not found, skipping yarn commands." + fi + + - name: Run frontend tests + run: | + yarn vitest + mv coverage/frontend/coverage.xml feature_branch_frontend_coverage.xml + + - name: Check for missing migrations + run: | + python manage.py makemigrations --check + + - name: Ensure previous Python coverage data is erased + run: | + coverage erase + + - name: Run Python unit tests + run: | + python -W default::DeprecationWarning -m coverage run manage.py test tests --settings="tests.test_settings" + + - name: Generate Python report coverage + run: | + coverage report + coverage json + mv coverage/python/coverage.json feature_branch_python_coverage.json + + - name: Upload frontend coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: feature-branch-frontend-coverage-report + path: feature_branch_frontend_coverage.xml + overwrite: true + + - name: Upload Python coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: feature-branch-python-coverage-report + path: feature_branch_python_coverage.json + overwrite: true + + build_target_branch: + runs-on: ubuntu-latest + + services: + postgres: + image: postgis/postgis:13-3.0 + env: + POSTGRES_PASSWORD: postgis + POSTGRES_DB: ${{ github.event.repository.name }} + ports: + - 5432:5432 + options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + + strategy: + fail-fast: false + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.ref }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + check-latest: true + + - name: Install Java, GDAL, and other system dependencies + run: | + sudo apt update + sudo apt-get install libxml2-dev libpq-dev openjdk-8-jdk libgdal-dev libxslt-dev + echo Postgres and ES dependencies installed + + - name: Install Python packages + run: | + python -m pip install --upgrade pip + pip install . + pip install -r ${{ github.event.repository.name }}/install/requirements.txt + pip install -r ${{ github.event.repository.name }}/install/requirements_dev.txt + echo Python packages installed + + - uses: ankane/setup-elasticsearch@v1 + with: + elasticsearch-version: 8 + + - name: Webpack frontend files + run: | + echo "Checking for yarn.lock file..." + if [ -f yarn.lock ]; then + echo "Removing yarn.lock due to yarn v1 package resolution issues" + echo "https://github.com/iarna/wide-align/issues/63" + rm yarn.lock + else + echo "yarn.lock not found, skipping remove." + fi + + echo "Checking for package.json..." + if [ -f package.json ]; then + echo "package.json found, building static bundle." + yarn && yarn build_test + else + echo "package.json not found, skipping yarn commands." + fi + + - name: Run frontend tests + run: | + if [ -f vitest.config.json ]; then + yarn vitest + mv coverage/frontend/coverage.xml target_branch_frontend_coverage.xml + else + echo "Unable to find vitest config. Skipping frontend tests." + fi + + - name: Check for missing migrations + run: | + python manage.py makemigrations --check + + - name: Ensure previous Python coverage data is erased + run: | + coverage erase + + - name: Run Python unit tests + run: | + python -W default::DeprecationWarning -m coverage run manage.py test tests --settings="tests.test_settings" + + - name: Generate Python report coverage + run: | + coverage report + coverage json + + # handles older target branch + if [ -f coverage/python/coverage.json ]; then + mv coverage/python/coverage.json target_branch_python_coverage.json + else + mv coverage.json target_branch_python_coverage.json + fi + + - name: Upload frontend coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: target-branch-frontend-coverage-report + path: target_branch_frontend_coverage.xml + overwrite: true + + - name: Upload Python coverage report as artifact + uses: actions/upload-artifact@v4 + with: + name: target-branch-python-coverage-report + path: target_branch_python_coverage.json + overwrite: true + + check_frontend_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch frontend coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-frontend-coverage-report + path: . + + - name: Extract feature branch frontend coverage data + shell: pwsh + run: | + [xml]$xml = Get-Content feature_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $feature_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $feature_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "feature_branch_frontend_coverage=$feature_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Download target branch frontend coverage report artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: target-branch-frontend-coverage-report + path: . + + - name: Check if target branch frontend coverage report artifact exists + run: | + if [ -f target_branch_frontend_coverage.xml ]; then + echo "target_branch_frontend_coverage_artifact_exists=true" >> $GITHUB_ENV + else + echo "Target branch coverage not found. Defaulting to 0% coverage." + echo "target_branch_frontend_coverage_artifact_exists=false" >> $GITHUB_ENV + fi + + - name: Extract target branch frontend coverage data + if: ${{ env.target_branch_frontend_coverage_artifact_exists == 'true' }} + shell: pwsh + run: | + [xml]$xml = Get-Content target_branch_frontend_coverage.xml + $metrics = $xml.coverage.project.metrics + + $statements = [double]$metrics.statements + $coveredstatements = [double]$metrics.coveredstatements + $conditionals = [double]$metrics.conditionals + $coveredconditionals = [double]$metrics.coveredconditionals + $methods = [double]$metrics.methods + $coveredmethods = [double]$metrics.coveredmethods + $elements = [double]$metrics.elements + $coveredelements = [double]$metrics.coveredelements + + $statement_coverage = 0.0 + $conditional_coverage = 0.0 + $method_coverage = 0.0 + $element_coverage = 0.0 + + if ($statements -gt 0) { + $statement_coverage = ($coveredstatements / $statements) * 100 + } + if ($conditionals -gt 0) { + $conditional_coverage = ($coveredconditionals / $conditionals) * 100 + } + if ($methods -gt 0) { + $method_coverage = ($coveredmethods / $methods) * 100 + } + if ($elements -gt 0) { + $element_coverage = ($coveredelements / $elements) * 100 + } + + $nonZeroCount = 0 + $totalCoverage = 0.0 + + if ($statements -gt 0) { $nonZeroCount++; $totalCoverage += $statement_coverage } + if ($conditionals -gt 0) { $nonZeroCount++; $totalCoverage += $conditional_coverage } + if ($methods -gt 0) { $nonZeroCount++; $totalCoverage += $method_coverage } + if ($elements -gt 0) { $nonZeroCount++; $totalCoverage += $element_coverage } + + $target_branch_frontend_coverage = 0.0 + if ($nonZeroCount -gt 0) { + $target_branch_frontend_coverage = $totalCoverage / $nonZeroCount + } + + Write-Output "target_branch_frontend_coverage=$target_branch_frontend_coverage" | Out-File -Append $env:GITHUB_ENV + + - name: Compare frontend feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_frontend_coverage=${feature_branch_frontend_coverage} + target_branch_frontend_coverage=${target_branch_frontend_coverage:-0.0} + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_frontend_coverage" -v target="$target_branch_frontend_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_frontend_coverage% to $feature_branch_frontend_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_frontend_coverage%) >= Target branch coverage ($target_branch_frontend_coverage%)." + fi + + check_python_coverage: + runs-on: ubuntu-latest + needs: [build_feature_branch, build_target_branch] + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' # Use the latest available version + check-latest: true + + - name: Download feature branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: feature-branch-python-coverage-report + path: . + + - name: Download target branch Python coverage report artifact + uses: actions/download-artifact@v4 + with: + name: target-branch-python-coverage-report + path: . + + - name: Compare Python feature coverage with target coverage + if: github.event_name == 'pull_request' + run: | + feature_branch_python_coverage=$(cat feature_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + target_branch_python_coverage=$(cat target_branch_python_coverage.json | grep -o '"totals": {[^}]*' | grep -o '"percent_covered": [0-9.]*' | awk -F ': ' '{print $2}') + + # Compare feature coverage with target coverage using floating-point comparison + if awk -v feature="$feature_branch_python_coverage" -v target="$target_branch_python_coverage" 'BEGIN { exit (feature < target) ? 0 : 1 }'; then + echo "Coverage decreased from $target_branch_python_coverage% to $feature_branch_python_coverage%. Please add or update tests to increase coverage." + exit 1 + else + echo "Feature branch coverage ($feature_branch_python_coverage%) >= Target branch coverage ($target_branch_python_coverage%)." + fi diff --git a/arches/install/arches-templates/.gitignore b/arches/install/arches-templates/.gitignore index 31a01364766..f9b6c762dc6 100644 --- a/arches/install/arches-templates/.gitignore +++ b/arches/install/arches-templates/.gitignore @@ -1,16 +1,16 @@ *.pyc *.log +node_modules +*.coverage {{ project_name }}/logs {{ project_name }}/export_deliverables {{ project_name }}/cantaloupe/* {{ project_name }}/staticfiles {{ project_name }}/media/packages -{{ project_name }}/media/node_modules {{ project_name }}/media/build/ {{ project_name }}/uploadedfiles/* {{ project_name }}/settings_local.py -{{ project_name }}/webpack/webpack-stats.json -{{ project_name }}/webpack/webpack-user-config.js +webpack-stats.json .vscode/ *.egg-info .DS_STORE diff --git a/arches/install/arches-templates/.yarnrc b/arches/install/arches-templates/.yarnrc deleted file mode 100644 index ca9e91e0448..00000000000 --- a/arches/install/arches-templates/.yarnrc +++ /dev/null @@ -1,2 +0,0 @@ ---install.modules-folder "./{{ project_name }}/media/node_modules" ---add.modules-folder "./{{ project_name }}/media/node_modules" \ No newline at end of file diff --git a/arches/install/arches-templates/eslint.config.mjs b/arches/install/arches-templates/eslint.config.mjs new file mode 100644 index 00000000000..4c919b1ca41 --- /dev/null +++ b/arches/install/arches-templates/eslint.config.mjs @@ -0,0 +1,42 @@ + +import js from "@eslint/js"; +import pluginVue from 'eslint-plugin-vue'; +import tseslint from 'typescript-eslint'; + +import vueESLintParser from 'vue-eslint-parser'; + +export default [ + js.configs.recommended, + ...pluginVue.configs['flat/recommended'], + ...tseslint.configs.recommended, + { + "languageOptions": { + "globals": { + "define": false, + "require": false, + "window": false, + "console": false, + "history": false, + "location": false, + "Promise": false, + "setTimeout": false, + "URL": false, + "URLSearchParams": false, + "fetch": false + }, + "parser": vueESLintParser, + "parserOptions": { + "ecmaVersion": 11, + "sourceType": "module", + "requireConfigFile": false, + "parser": { + "ts": "@typescript-eslint/parser" + } + }, + }, + "rules": { + "semi": ["error", "always"], + "vue/html-indent": ["error", 4] + }, + }, +] \ No newline at end of file diff --git a/arches/install/arches-templates/gettext.config.js b/arches/install/arches-templates/gettext.config.js index c1289b98bd2..27934951e4f 100644 --- a/arches/install/arches-templates/gettext.config.js +++ b/arches/install/arches-templates/gettext.config.js @@ -1,43 +1,43 @@ module.exports = { - input: { - path: "./{{ project_name }}/src", // only files in this directory are considered for extraction - include: ["**/*.vue", "**/*.ts"], // glob patterns to select files for extraction - exclude: [], // glob patterns to exclude files from extraction - jsExtractorOpts:[ // custom extractor keyword. default empty. - { - keyword: "__", // only extractor default keyword such as $gettext,use keyword to custom - options: { // see https://github.com/lukasgeiter/gettext-extractor - content: { - replaceNewLines: "\n", - }, - arguments: { - text: 0, - }, - }, + input: { + path: "./{{ project_name }}/src", // only files in this directory are considered for extraction + include: ["**/*.vue", "**/*.ts"], // glob patterns to select files for extraction + exclude: [], // glob patterns to exclude files from extraction + jsExtractorOpts: [ // custom extractor keyword. default empty. + { + keyword: "__", // only extractor default keyword such as $gettext,use keyword to custom + options: { // see https://github.com/lukasgeiter/gettext-extractor + content: { + replaceNewLines: "\n", }, - { - keyword: "_n", // $ngettext - options: { - content: { - replaceNewLines: "\n", - }, - arguments: { - text: 0, - textPlural: 1, - }, - }, + arguments: { + text: 0, }, - ], - compileTemplate: false, // do not compile