From e86196a5b286871f8121299cc996b80a5e1c6ae1 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 17 Oct 2024 11:35:13 +0200 Subject: [PATCH 01/41] [PAGOPA-2282] chore: Update ingress timeout --- helm/values-uat.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 04fa45b8..119bb668 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -94,11 +94,11 @@ microservice-chart: create: true host: "weuuat.fdr.internal.uat.platform.pagopa.it" path: /pagopa-fdr-service(/|$)(.*) - # currently set ingress timeout to 5m, until async handling will be introduced + # currently set ingress timeout to 50m, until async handling will be introduced annotations: - nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" - nginx.ingress.kubernetes.io/proxy-send-timeout: "300" - nginx.ingress.kubernetes.io/send_timeout: "300" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "3000" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3000" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3000" + nginx.ingress.kubernetes.io/send_timeout: "3000" canaryDelivery: create: false From ded78ff411ad0fb64f6cdcb3cb40b246a5abb445 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 17 Oct 2024 12:46:30 +0200 Subject: [PATCH 02/41] [PAGOPA-2282] chore: Add log --- src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 345806d9..bc9863f9 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -319,7 +319,10 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.setStatus(FdrStatusEnumEntity.PUBLISHED); List fdrPaymentPublishEntities = mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); + + log.info("Starting persistent storage on Mongo of FDR payment entities"); FdrPaymentPublishEntity.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); + log.info("End of persistent storage on Mongo of FDR payment entities"); // salva su storage dello storico HistoryBlobBody body = historyService.saveJsonFile(fdrPublishEntity, fdrPaymentPublishEntities); From e3285ce56773200eba6963ba5a6bc69d43ed289c Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 17 Oct 2024 13:02:31 +0200 Subject: [PATCH 03/41] [PAGOPA-2282] chore: Add LOG_LEVEL env --- helm/values-dev.yaml | 1 + helm/values-uat.yaml | 1 + src/main/resources/application.properties | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index e6c0d4bc..5b7ddf53 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -24,6 +24,7 @@ microservice-chart: APP_ENVIRONMENT: "dev" TZ: "Europe/Rome" OTEL_ENDPOINT: "http://otel-collector.elastic-system.svc.cluster.local:4317" + LOG_LEVEL: "INFO" ADAPTER_API_CONFIG_CACHE_URL: "https://api.dev.platform.pagopa.it/api-config-cache/p/v1" ADAPTER_API_CONFIG_CACHE_CRON: "0 */1 * * * ?" QUEUE_CONVERSION_NAME: "flowidsendqueue" diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 119bb668..78e816f7 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -24,6 +24,7 @@ microservice-chart: APP_ENVIRONMENT: "uat" TZ: "Europe/Rome" OTEL_ENDPOINT: "http://otel-collector.elastic-system.svc.cluster.local:4317" + LOG_LEVEL: "INFO" ADAPTER_API_CONFIG_CACHE_URL: "https://api.uat.platform.pagopa.it/api-config-cache/p/v1" ADAPTER_API_CONFIG_CACHE_CRON: "0 */1 * * * ?" QUEUE_CONVERSION_NAME: "flowidsendqueue" diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index d4fcca20..83107936 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -17,7 +17,7 @@ quarkus.locales=it-IT ################### quarkus.log.file.json=false quarkus.log.level=INFO -quarkus.log.category."it.gov.pagopa.fdr".level=DEBUG +quarkus.log.category."it.gov.pagopa.fdr".level=${LOG_LEVEL:INFO} %dev.quarkus.log.console.json=false %test.quarkus.log.console.json=false From 4828a8f375d416db38885e5587adf0b540d8149e Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 17 Oct 2024 15:36:40 +0200 Subject: [PATCH 04/41] [PAGOPA-2282] chore: Add info log --- .../java/it/gov/pagopa/fdr/service/psps/PspsService.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index bc9863f9..68d5aa84 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -325,11 +325,16 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern log.info("End of persistent storage on Mongo of FDR payment entities"); // salva su storage dello storico + log.info("Starting saveJsonFile storage on BlobStorage of FDR payment entities"); HistoryBlobBody body = historyService.saveJsonFile(fdrPublishEntity, fdrPaymentPublishEntities); + log.info("End of saveJsonFile storage on BlobStorage of FDR payment entities"); + fdrPublishEntity.setRefJson(body); fdrPublishEntity.persistEntity(); + log.info("Starting saveOnStorage storage on BlobStorage of FDR payment entities"); historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); log.debug("Delete FdrInsertEntity"); fdrEntity.delete(); From a5bcc35e2102fc25a041dfc42e374f19c3990abc Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 24 Oct 2024 16:32:40 +0200 Subject: [PATCH 05/41] [PAGOPA-2282] chore: Config vert.x event-loop-execute-time --- src/main/resources/application.properties | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 83107936..ca678d81 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -69,6 +69,11 @@ quarkus.mongodb.connection-string=${MONGODB_CONNECTION_STRING:${mockserver.mongo %dev.quarkus.mongodb.connection-string=mongodb://root:example@localhost:27017/ quarkus.mongodb.database=fdr +################### +## Vert.x +################### +quarkus.vertx.max-event-loop-execute-time=60s + ################### ## ADAPTER API CONFIG CACHE ################### From 8349e18a68fb5f402f637220865975ab41f4dacb Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 24 Oct 2024 17:11:58 +0200 Subject: [PATCH 06/41] [PAGOPA-2282] ci: Update release action --- .github/workflows/04_release_and_deploy.yml | 57 +++++++++++++++------ 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/.github/workflows/04_release_and_deploy.yml b/.github/workflows/04_release_and_deploy.yml index f9432be9..5b16879d 100644 --- a/.github/workflows/04_release_and_deploy.yml +++ b/.github/workflows/04_release_and_deploy.yml @@ -18,15 +18,13 @@ on: - dev - uat - prod - semver: + version: required: false type: choice description: Select the version options: - '' - - skip - - promote - - patch + - skip_or_promote - new_release - breaking_change beta: @@ -40,10 +38,6 @@ on: environment: required: true type: string - semver: - required: true - type: string - default: skip permissions: packages: write @@ -59,14 +53,47 @@ jobs: name: Setup runs-on: ubuntu-latest outputs: - semver: ${{ steps.semver_setup.outputs.semver }} - environment: ${{ steps.semver_setup.outputs.environment }} + semver: ${{ steps.get_semver.outputs.semver }} + environment: ${{ steps.get_env.outputs.environment }} steps: - - name: Semver setup - id: semver_setup - uses: pagopa/github-actions-template/nodo5-semver-setup@ce252c8501c9242bd6045f7cdd650736b2f38777 - with: - semver: ${{ inputs.semver }} + - name: pull request rejected + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged != true + run: | + echo "❌ PR was closed without a merge" + exit 1 + + # Set Semvar + - run: echo "SEMVER=patch" >> $GITHUB_ENV + + - if: ${{ (github.event.pull_request.merged && contains(github.event.pull_request.labels.*.name, 'breaking-change')) }} + run: echo "SEMVER=major" >> $GITHUB_ENV + + # force semver if dev, !=main or skip release + - if: ${{ inputs.version == 'new_release' }} + run: echo "SEMVER=minor" >> $GITHUB_ENV + + - if: ${{ inputs.version == 'breaking_change' }} + run: echo "SEMVER=major" >> $GITHUB_ENV + + - if: ${{ github.ref_name != 'main' }} + run: echo "SEMVER=buildNumber" >> $GITHUB_ENV + + - if: ${{ inputs.version == 'skip_or_promote' }} + run: echo "SEMVER=skip" >> $GITHUB_ENV + + - id: get_semver + name: Set Output + run: echo "semver=${{env.SEMVER}}" >> $GITHUB_OUTPUT + + # Set Environment + - run: echo "ENVIRNOMENT=${{ inputs.environment}}" >> $GITHUB_ENV + + - if: ${{ inputs.environment == null }} + run: echo "ENVIRNOMENT=dev" >> $GITHUB_ENV + + - id: get_env + name: Set Output + run: echo "environment=${{env.ENVIRNOMENT}}" >> $GITHUB_OUTPUT release: needs: [setup] From 237b6a01b2dd34a8e754a03fbd9d4ce4781429b7 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Thu, 24 Oct 2024 15:13:22 +0000 Subject: [PATCH 07/41] Bump to version 1.0.21-1-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index cfbe48b9..0d4d2e95 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.28.0" -appVersion: "1.0.21" +version: "1.29.0" +appVersion: "1.0.21-1-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 5b7ddf53..bca220de 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21 + tag: 1.0.21-1-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 78e816f7..bcc0e0e8 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21 + tag: 1.0.21-1-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 45a5a1a2..9c682261 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21" + "version": "1.0.21-1-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index ff2b3bdd..83892a48 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21" + "version": "1.0.21-1-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 7d98681b..0435736c 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21" + "version": "1.0.21-1-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index afffd592..dc794d77 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21 + 1.0.21-1-PAGOPA-2282 3.11.0 1.18.26 From 1c38b5f656c319a72cb51e356be303db9d0eafcf Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Thu, 24 Oct 2024 18:27:12 +0200 Subject: [PATCH 08/41] [PAGOPA-2282] chore: vert.x --- .../fdr/FdrPaymentPublishEntity.java | 37 ++++++++++--------- src/main/resources/application.properties | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java index b1493f1d..02f2f899 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java @@ -6,6 +6,7 @@ import io.quarkus.mongodb.panache.common.MongoEntity; import io.quarkus.panache.common.Parameters; import io.quarkus.panache.common.Sort; +import io.vertx.core.Vertx; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; import java.util.List; @@ -99,22 +100,24 @@ public static PanacheQuery findByPspAndIuvIur( return find(query, sort, params); } - // public static PanacheQuery findByFdrAndPspId( - // String fdr, String pspId, Sort sort) { - // return find( - // "ref_fdr = :fdr and ref_fdr_sender_psp_id = :pspId", - // sort, - // Parameters.with("fdr", fdr).and("pspId", pspId).map()); - // } - // - // public static long deleteByFdrAndPspId(String fdr, String pspId) { - // return delete( - // "ref_fdr = :fdr and ref_fdr_sender_psp_id = :pspId", - // Parameters.with("fdr", fdr).and("pspId", pspId).map()); - // } - // - public static void persistFdrPaymentPublishEntities( - List fdrPaymentPublishEntities) { - persist(fdrPaymentPublishEntities); + // persists a list of FdrPaymentPublishEntity to mongo using non-blocking vert.x + public static void persistFdrPaymentPublishEntities(List entities) { + Vertx vertx = Vertx.vertx(); + vertx.executeBlocking( + promise -> { + try { + persist(entities); + promise.complete(); + } catch (Exception e) { + promise.fail(e); + } + }, + res -> { + if (res.succeeded()) { + // Handle success if needed + } else { + // Handle failure if needed + } + }); } } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index ca678d81..da9a9a02 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -72,7 +72,7 @@ quarkus.mongodb.database=fdr ################### ## Vert.x ################### -quarkus.vertx.max-event-loop-execute-time=60s +# quarkus.vertx.max-event-loop-execute-time=2s ################### ## ADAPTER API CONFIG CACHE From 5728adc5e004f07e947bd6314ce7f85c1322530b Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Thu, 24 Oct 2024 16:28:44 +0000 Subject: [PATCH 09/41] Bump to version 1.0.21-2-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 0d4d2e95..fb2a3d7c 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.29.0" -appVersion: "1.0.21-1-PAGOPA-2282" +version: "1.30.0" +appVersion: "1.0.21-2-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index bca220de..5ef06174 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-1-PAGOPA-2282 + tag: 1.0.21-2-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index bcc0e0e8..b6a108de 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-1-PAGOPA-2282 + tag: 1.0.21-2-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 9c682261..21b9bd1d 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-1-PAGOPA-2282" + "version": "1.0.21-2-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 83892a48..9dcd8262 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-1-PAGOPA-2282" + "version": "1.0.21-2-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 0435736c..ca9fc8cd 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-1-PAGOPA-2282" + "version": "1.0.21-2-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index dc794d77..c7caa6cf 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-1-PAGOPA-2282 + 1.0.21-2-PAGOPA-2282 3.11.0 1.18.26 From ace4a0ba969a3d5637a6b61843b53925e3e75529 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 12:26:35 +0200 Subject: [PATCH 10/41] [PAGOPA-2282] chore: Add reactive entity --- .../fdr/FdrPaymentPublishEntity.java | 22 ------------------- .../fdr/FdrPaymentPublishEntityReactive.java | 20 +++++++++++++++++ .../pagopa/fdr/service/psps/PspsService.java | 7 ++---- 3 files changed, 22 insertions(+), 27 deletions(-) create mode 100644 src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java index 02f2f899..26e5026e 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java @@ -6,7 +6,6 @@ import io.quarkus.mongodb.panache.common.MongoEntity; import io.quarkus.panache.common.Parameters; import io.quarkus.panache.common.Sort; -import io.vertx.core.Vertx; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; import java.util.List; @@ -99,25 +98,4 @@ public static PanacheQuery findByPspAndIuvIur( } return find(query, sort, params); } - - // persists a list of FdrPaymentPublishEntity to mongo using non-blocking vert.x - public static void persistFdrPaymentPublishEntities(List entities) { - Vertx vertx = Vertx.vertx(); - vertx.executeBlocking( - promise -> { - try { - persist(entities); - promise.complete(); - } catch (Exception e) { - promise.fail(e); - } - }, - res -> { - if (res.succeeded()) { - // Handle success if needed - } else { - // Handle failure if needed - } - }); - } } diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java new file mode 100644 index 00000000..b6bde95a --- /dev/null +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java @@ -0,0 +1,20 @@ +package it.gov.pagopa.fdr.repository.fdr; + +import io.quarkus.mongodb.panache.common.MongoEntity; +import io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoEntity; +import io.smallrye.mutiny.Uni; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@Data +@EqualsAndHashCode(callSuper = true) +@MongoEntity(collection = "fdr_payment_publish") +public class FdrPaymentPublishEntityReactive extends ReactivePanacheMongoEntity { + + // Reactive persistence method for FdrPaymentPublishEntity objects + public static Uni persistFdrPaymentPublishEntities(List entities) { + return persist(entities); + } +} diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 68d5aa84..32992514 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -10,10 +10,7 @@ import io.quarkus.panache.common.Sort; import it.gov.pagopa.fdr.exception.AppErrorCodeMessageEnum; import it.gov.pagopa.fdr.exception.AppException; -import it.gov.pagopa.fdr.repository.fdr.FdrInsertEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPaymentInsertEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPaymentPublishEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPublishEntity; +import it.gov.pagopa.fdr.repository.fdr.*; import it.gov.pagopa.fdr.repository.fdr.model.FdrStatusEnumEntity; import it.gov.pagopa.fdr.repository.fdr.projection.FdrInsertProjection; import it.gov.pagopa.fdr.repository.fdr.projection.FdrPublishByPspProjection; @@ -321,7 +318,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); log.info("Starting persistent storage on Mongo of FDR payment entities"); - FdrPaymentPublishEntity.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); + FdrPaymentPublishEntityReactive.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); log.info("End of persistent storage on Mongo of FDR payment entities"); // salva su storage dello storico From f9cc033a2566e298c68f17a4a9bb715b152be5a4 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 10:28:42 +0000 Subject: [PATCH 11/41] Bump to version 1.0.21-3-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index fb2a3d7c..9d6c5072 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.30.0" -appVersion: "1.0.21-2-PAGOPA-2282" +version: "1.31.0" +appVersion: "1.0.21-3-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 5ef06174..0c0bdee2 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-2-PAGOPA-2282 + tag: 1.0.21-3-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index b6a108de..d7b11f07 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-2-PAGOPA-2282 + tag: 1.0.21-3-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 21b9bd1d..5e1db142 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-2-PAGOPA-2282" + "version": "1.0.21-3-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 9dcd8262..4644429a 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-2-PAGOPA-2282" + "version": "1.0.21-3-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index ca9fc8cd..ce1544ef 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-2-PAGOPA-2282" + "version": "1.0.21-3-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index c7caa6cf..fc5f9b9d 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-2-PAGOPA-2282 + 1.0.21-3-PAGOPA-2282 3.11.0 1.18.26 From 537883fe01db5fd4b5cc0bd7e3f2a0330de9acb4 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 14:54:10 +0200 Subject: [PATCH 12/41] [PAGOPA-2282] refactoring: Add async saveOnStorage --- .../pagopa/fdr/service/history/HistoryService.java | 14 ++++++++++++++ .../gov/pagopa/fdr/service/psps/PspsService.java | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java index 941086a3..b462e379 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.time.Instant; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; @@ -107,6 +108,19 @@ public void saveOnStorage( } } + public void asyncSaveOnStorage(FdrPublishEntity fdrEntity, List paymentsList) { + CompletableFuture executeSaveOnStorage = CompletableFuture.supplyAsync(() -> { + this.saveOnStorage(fdrEntity, paymentsList); + return true; + }); + executeSaveOnStorage + .thenAccept(value -> logger.debugf("Async saveOnStorage successful executed")) + .exceptionally(e -> { + logger.error("Exception during async saveOnStorage: ", e); + return null; + }); + } + public HistoryBlobBody saveJsonFile( FdrPublishEntity fdrEntity, List paymentsList) { if (blobContainerClient != null) { diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 32992514..6f7f3042 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -330,7 +330,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.persistEntity(); log.info("Starting saveOnStorage storage on BlobStorage of FDR payment entities"); - historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + historyService.asyncSaveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); log.debug("Delete FdrInsertEntity"); From 66149185a68e85b1213c1ac9f38e9a5e76f4a449 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 12:55:40 +0000 Subject: [PATCH 13/41] Bump to version 1.0.21-4-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 9d6c5072..48dc791f 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.31.0" -appVersion: "1.0.21-3-PAGOPA-2282" +version: "1.32.0" +appVersion: "1.0.21-4-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 0c0bdee2..56281b3c 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-3-PAGOPA-2282 + tag: 1.0.21-4-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index d7b11f07..46299d0e 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-3-PAGOPA-2282 + tag: 1.0.21-4-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 5e1db142..cd3b5bbe 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-3-PAGOPA-2282" + "version": "1.0.21-4-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 4644429a..98129de0 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-3-PAGOPA-2282" + "version": "1.0.21-4-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index ce1544ef..ac4ff934 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-3-PAGOPA-2282" + "version": "1.0.21-4-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index fc5f9b9d..61af6f4c 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-3-PAGOPA-2282 + 1.0.21-4-PAGOPA-2282 3.11.0 1.18.26 From 2c50825099e15e4537b3cd964c7ab17249231c55 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 16:14:56 +0200 Subject: [PATCH 14/41] [PAGOPA-2282] refactoring --- .../it/gov/pagopa/fdr/service/psps/PspsService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 6f7f3042..a8a7bc16 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -333,17 +333,18 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern historyService.asyncSaveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - log.debug("Delete FdrInsertEntity"); + log.info("Starting delete FdrInsertEntity"); fdrEntity.delete(); - log.debugf( + log.infof( "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); + log.info("End delete deleteByFdrAndPspId"); // add to conversion queue if (internalPublish) { - log.debug("NOT Add FdrInsertEntity in queue fdr message"); + log.info("NOT Add FdrInsertEntity in queue fdr message"); } else { - log.debug("Add FdrInsertEntity in queue fdr message"); + log.info("Starting add FdrInsertEntity in queue fdr message"); conversionQueue.addQueueFlowMessage( FdrMessage.builder() .fdr(fdrEntity.getFdr()) @@ -352,6 +353,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern .retry(0L) .revision(fdrEntity.getRevision()) .build()); + log.info("End add FdrInsertEntity in queue fdr message"); } String sessionId = (String) MDC.get(TRX_ID); From 2ca8902c06af6aae7a0be47fc053ced9254dc3b8 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 14:16:15 +0000 Subject: [PATCH 15/41] Bump to version 1.0.21-5-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 48dc791f..9fa24736 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.32.0" -appVersion: "1.0.21-4-PAGOPA-2282" +version: "1.33.0" +appVersion: "1.0.21-5-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 56281b3c..7f8663b2 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-4-PAGOPA-2282 + tag: 1.0.21-5-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 46299d0e..46b67a03 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-4-PAGOPA-2282 + tag: 1.0.21-5-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index cd3b5bbe..a9a70119 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-4-PAGOPA-2282" + "version": "1.0.21-5-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 98129de0..d3e8faef 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-4-PAGOPA-2282" + "version": "1.0.21-5-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index ac4ff934..8efebec7 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-4-PAGOPA-2282" + "version": "1.0.21-5-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 61af6f4c..f1860b72 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-4-PAGOPA-2282 + 1.0.21-5-PAGOPA-2282 3.11.0 1.18.26 From 556daf056e202989ad510cabc3c46c392424d987 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 16:35:52 +0200 Subject: [PATCH 16/41] [PAGOPA-2282] refactoring --- .../pagopa/fdr/service/psps/PspsService.java | 81 ++++++++++++------- 1 file changed, 52 insertions(+), 29 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index a8a7bc16..7d637e6a 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -32,6 +32,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import org.jboss.logging.Logger; import org.jboss.logging.MDC; @@ -330,50 +331,72 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.persistEntity(); log.info("Starting saveOnStorage storage on BlobStorage of FDR payment entities"); - historyService.asyncSaveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); - log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - - log.info("Starting delete FdrInsertEntity"); - fdrEntity.delete(); - log.infof( - "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); - FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); - log.info("End delete deleteByFdrAndPspId"); + CompletableFuture executeSaveOnStorage = CompletableFuture.supplyAsync(() -> { + historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + log.info("Async saveOnStorage successful executed"); + return true; + }); + executeSaveOnStorage + .thenAccept(value -> { + log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); + + log.info("Starting delete FdrInsertEntity"); + fdrEntity.delete(); + log.infof( + "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); + FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); + log.info("End delete deleteByFdrAndPspId"); + log.info("Starting conversion to queue"); + this.addToConversionQueue(internalPublish, fdrEntity); + log.info("End of conversion to queue"); + log.info("Starting write to re"); + this.rePublish(fdr, pspId, fdrPublishEntity); + log.info("End of write to re"); + }) + .exceptionally(e -> { + log.error("Exception during async saveOnStorage: ", e); + return null; + }); + } + private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { // add to conversion queue if (internalPublish) { log.info("NOT Add FdrInsertEntity in queue fdr message"); } else { log.info("Starting add FdrInsertEntity in queue fdr message"); conversionQueue.addQueueFlowMessage( - FdrMessage.builder() - .fdr(fdrEntity.getFdr()) - .pspId(fdrEntity.getSender().getPspId()) - .organizationId(fdrEntity.getReceiver().getOrganizationId()) - .retry(0L) - .revision(fdrEntity.getRevision()) - .build()); + FdrMessage.builder() + .fdr(fdrEntity.getFdr()) + .pspId(fdrEntity.getSender().getPspId()) + .organizationId(fdrEntity.getReceiver().getOrganizationId()) + .retry(0L) + .revision(fdrEntity.getRevision()) + .build()); log.info("End add FdrInsertEntity in queue fdr message"); } + } + private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEntity) { String sessionId = (String) MDC.get(TRX_ID); MDC.put(EVENT_CATEGORY, EventTypeEnum.INTERNAL.name()); reService.sendEvent( - ReInternal.builder() - .serviceIdentifier(AppVersionEnum.FDR003) - .created(Instant.now()) - .sessionId(sessionId) - .eventType(EventTypeEnum.INTERNAL) - .fdrPhysicalDelete(false) - .fdrStatus(FdrStatusEnum.PUBLISHED) - .fdr(fdr) - .pspId(pspId) - .organizationId(fdrPublishEntity.getReceiver().getOrganizationId()) - .revision(fdrPublishEntity.getRevision()) - .fdrAction(FdrActionEnum.PUBLISH) - .build()); + ReInternal.builder() + .serviceIdentifier(AppVersionEnum.FDR003) + .created(Instant.now()) + .sessionId(sessionId) + .eventType(EventTypeEnum.INTERNAL) + .fdrPhysicalDelete(false) + .fdrStatus(FdrStatusEnum.PUBLISHED) + .fdr(fdr) + .pspId(pspId) + .organizationId(fdrPublishEntity.getReceiver().getOrganizationId()) + .revision(fdrPublishEntity.getRevision()) + .fdrAction(FdrActionEnum.PUBLISH) + .build()); } + @WithSpan(kind = SERVER) public void deleteByFdr(String action, String pspId, String fdr) { log.infof(AppMessageUtil.logExecute(action)); From e883d33c510ed67afe510b54836ea91b0cd709a5 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 14:37:19 +0000 Subject: [PATCH 17/41] Bump to version 1.0.21-6-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 9fa24736..631ef125 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.33.0" -appVersion: "1.0.21-5-PAGOPA-2282" +version: "1.34.0" +appVersion: "1.0.21-6-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 7f8663b2..b8afbd93 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-5-PAGOPA-2282 + tag: 1.0.21-6-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 46b67a03..5c0405b5 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-5-PAGOPA-2282 + tag: 1.0.21-6-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index a9a70119..70020a5c 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-5-PAGOPA-2282" + "version": "1.0.21-6-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index d3e8faef..e564d0ec 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-5-PAGOPA-2282" + "version": "1.0.21-6-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 8efebec7..90a1e82e 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-5-PAGOPA-2282" + "version": "1.0.21-6-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index f1860b72..032e85ba 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-5-PAGOPA-2282 + 1.0.21-6-PAGOPA-2282 3.11.0 1.18.26 From b7b15ea9befbb4d8bed8567573a84bb5dd090016 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 17:18:23 +0200 Subject: [PATCH 18/41] [PAGOPA-2282] refactoring: Remove log --- .../it/gov/pagopa/fdr/service/psps/PspsService.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 7d637e6a..7cfc452e 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -323,9 +323,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern log.info("End of persistent storage on Mongo of FDR payment entities"); // salva su storage dello storico - log.info("Starting saveJsonFile storage on BlobStorage of FDR payment entities"); HistoryBlobBody body = historyService.saveJsonFile(fdrPublishEntity, fdrPaymentPublishEntities); - log.info("End of saveJsonFile storage on BlobStorage of FDR payment entities"); fdrPublishEntity.setRefJson(body); fdrPublishEntity.persistEntity(); @@ -339,19 +337,13 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern executeSaveOnStorage .thenAccept(value -> { log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - - log.info("Starting delete FdrInsertEntity"); fdrEntity.delete(); log.infof( "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); log.info("End delete deleteByFdrAndPspId"); - log.info("Starting conversion to queue"); this.addToConversionQueue(internalPublish, fdrEntity); - log.info("End of conversion to queue"); - log.info("Starting write to re"); this.rePublish(fdr, pspId, fdrPublishEntity); - log.info("End of write to re"); }) .exceptionally(e -> { log.error("Exception during async saveOnStorage: ", e); @@ -378,6 +370,7 @@ private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEn } private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEntity) { + log.info("Starting write to re"); String sessionId = (String) MDC.get(TRX_ID); MDC.put(EVENT_CATEGORY, EventTypeEnum.INTERNAL.name()); reService.sendEvent( @@ -394,6 +387,7 @@ private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEnti .revision(fdrPublishEntity.getRevision()) .fdrAction(FdrActionEnum.PUBLISH) .build()); + log.info("End write to re"); } From 36ead2cd540f2df90ead8ca035768e1ce39e94db Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 15:19:56 +0000 Subject: [PATCH 19/41] Bump to version 1.0.21-7-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 631ef125..bba6cc82 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.34.0" -appVersion: "1.0.21-6-PAGOPA-2282" +version: "1.35.0" +appVersion: "1.0.21-7-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index b8afbd93..0510ca3a 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-6-PAGOPA-2282 + tag: 1.0.21-7-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 5c0405b5..f654a67f 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-6-PAGOPA-2282 + tag: 1.0.21-7-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 70020a5c..29fef0e3 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-6-PAGOPA-2282" + "version": "1.0.21-7-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index e564d0ec..726eb450 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-6-PAGOPA-2282" + "version": "1.0.21-7-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 90a1e82e..ebed5329 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-6-PAGOPA-2282" + "version": "1.0.21-7-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 032e85ba..94c80a2a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-6-PAGOPA-2282 + 1.0.21-7-PAGOPA-2282 3.11.0 1.18.26 From fd599f3e0f3551cd0bd0e7997fac57473e1db0f2 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Fri, 25 Oct 2024 17:23:12 +0200 Subject: [PATCH 20/41] Fix code scanning alert no. 717: Log Injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 7cfc452e..f1b0bb37 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -338,8 +338,9 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern .thenAccept(value -> { log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); fdrEntity.delete(); + String sanitizedFdr = fdr.replace("\n", "").replace("\r", ""); log.infof( - "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); + "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), sanitizedFdr, pspId); FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); log.info("End delete deleteByFdrAndPspId"); this.addToConversionQueue(internalPublish, fdrEntity); From f596fc524f66d5b80fa993ef134255f955f78910 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 15:36:03 +0000 Subject: [PATCH 21/41] Bump to version 1.0.21-8-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index bba6cc82..96b4810a 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.35.0" -appVersion: "1.0.21-7-PAGOPA-2282" +version: "1.36.0" +appVersion: "1.0.21-8-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 0510ca3a..61f9c9ef 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-7-PAGOPA-2282 + tag: 1.0.21-8-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index f654a67f..e6a8a4c4 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-7-PAGOPA-2282 + tag: 1.0.21-8-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 29fef0e3..a95a3811 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-7-PAGOPA-2282" + "version": "1.0.21-8-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 726eb450..d99849c0 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-7-PAGOPA-2282" + "version": "1.0.21-8-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index ebed5329..690660e5 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-7-PAGOPA-2282" + "version": "1.0.21-8-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 94c80a2a..f865e046 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-7-PAGOPA-2282 + 1.0.21-8-PAGOPA-2282 3.11.0 1.18.26 From af70948feb882b3c7db274edf1fd3dc6de06d6a3 Mon Sep 17 00:00:00 2001 From: AngeloCaporaso Date: Fri, 25 Oct 2024 18:10:12 +0200 Subject: [PATCH 22/41] [PAGOPA-2282] refactoring --- .../fdr/service/history/HistoryService.java | 13 ------------- .../gov/pagopa/fdr/service/psps/PspsService.java | 15 +++++++++------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java index b462e379..ce245eb9 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java @@ -108,19 +108,6 @@ public void saveOnStorage( } } - public void asyncSaveOnStorage(FdrPublishEntity fdrEntity, List paymentsList) { - CompletableFuture executeSaveOnStorage = CompletableFuture.supplyAsync(() -> { - this.saveOnStorage(fdrEntity, paymentsList); - return true; - }); - executeSaveOnStorage - .thenAccept(value -> logger.debugf("Async saveOnStorage successful executed")) - .exceptionally(e -> { - logger.error("Exception during async saveOnStorage: ", e); - return null; - }); - } - public HistoryBlobBody saveJsonFile( FdrPublishEntity fdrEntity, List paymentsList) { if (blobContainerClient != null) { diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index f1b0bb37..97a2b420 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -337,13 +337,16 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern executeSaveOnStorage .thenAccept(value -> { log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - fdrEntity.delete(); - String sanitizedFdr = fdr.replace("\n", "").replace("\r", ""); - log.infof( - "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), sanitizedFdr, pspId); - FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); - log.info("End delete deleteByFdrAndPspId"); + this.addToConversionQueue(internalPublish, fdrEntity); + + // todo: handles deletes, check insert overwrites + // fdrEntity.delete(); +// log.infof( +// "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); + // FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); +// log.info("End delete deleteByFdrAndPspId"); + this.rePublish(fdr, pspId, fdrPublishEntity); }) .exceptionally(e -> { From f643956d8fe72ef7c4060f12cd7b9810062c4565 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Fri, 25 Oct 2024 16:11:40 +0000 Subject: [PATCH 23/41] Bump to version 1.0.21-9-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 96b4810a..ad937777 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.36.0" -appVersion: "1.0.21-8-PAGOPA-2282" +version: "1.37.0" +appVersion: "1.0.21-9-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 61f9c9ef..98b4bd5f 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-8-PAGOPA-2282 + tag: 1.0.21-9-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index e6a8a4c4..ec5714dc 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-8-PAGOPA-2282 + tag: 1.0.21-9-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index a95a3811..b41e27de 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-8-PAGOPA-2282" + "version": "1.0.21-9-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index d99849c0..b6c5149d 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-8-PAGOPA-2282" + "version": "1.0.21-9-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 690660e5..e706f269 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-8-PAGOPA-2282" + "version": "1.0.21-9-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index f865e046..1fdb24b4 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-8-PAGOPA-2282 + 1.0.21-9-PAGOPA-2282 3.11.0 1.18.26 From 816f8be8d070ce2af1be251a4aa9ef9cdc8ab25d Mon Sep 17 00:00:00 2001 From: Francesco Cesareo Date: Fri, 25 Oct 2024 20:00:30 +0200 Subject: [PATCH 24/41] draft of delete batch --- .../fdr/FdrPaymentPublishEntityReactive.java | 11 +- .../fdr/service/history/HistoryService.java | 1 - .../pagopa/fdr/service/psps/PspsService.java | 109 +++++++++++------- 3 files changed, 70 insertions(+), 51 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java index b6bde95a..ddf17536 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java @@ -3,18 +3,17 @@ import io.quarkus.mongodb.panache.common.MongoEntity; import io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoEntity; import io.smallrye.mutiny.Uni; +import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; -import java.util.List; - @Data @EqualsAndHashCode(callSuper = true) @MongoEntity(collection = "fdr_payment_publish") public class FdrPaymentPublishEntityReactive extends ReactivePanacheMongoEntity { - // Reactive persistence method for FdrPaymentPublishEntity objects - public static Uni persistFdrPaymentPublishEntities(List entities) { - return persist(entities); - } + // Reactive persistence method for FdrPaymentPublishEntity objects + public static Uni persistFdrPaymentPublishEntities(List entities) { + return persist(entities); + } } diff --git a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java index ce245eb9..941086a3 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/history/HistoryService.java @@ -33,7 +33,6 @@ import java.io.IOException; import java.time.Instant; import java.util.*; -import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 97a2b420..7f103a6f 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -34,6 +34,7 @@ import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.jboss.logging.Logger; import org.jboss.logging.MDC; @@ -329,30 +330,51 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.persistEntity(); log.info("Starting saveOnStorage storage on BlobStorage of FDR payment entities"); - CompletableFuture executeSaveOnStorage = CompletableFuture.supplyAsync(() -> { - historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); - log.info("Async saveOnStorage successful executed"); - return true; - }); + CompletableFuture executeSaveOnStorage = + CompletableFuture.supplyAsync( + () -> { + historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + log.info("Async saveOnStorage successful executed"); + return true; + }); executeSaveOnStorage - .thenAccept(value -> { - log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - - this.addToConversionQueue(internalPublish, fdrEntity); - - // todo: handles deletes, check insert overwrites - // fdrEntity.delete(); -// log.infof( -// "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); - // FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); -// log.info("End delete deleteByFdrAndPspId"); - - this.rePublish(fdr, pspId, fdrPublishEntity); - }) - .exceptionally(e -> { - log.error("Exception during async saveOnStorage: ", e); - return null; - }); + .thenAccept( + value -> { + log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); + + this.addToConversionQueue(internalPublish, fdrEntity); + + // todo: handles deletes, check insert overwrites + fdrEntity.delete(); + log.infof( + "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", + fdrEntity.getRevision(), fdr, pspId); + int batchSize = 1000; + List> batches = + IntStream.range(0, (paymentInsertEntities.size() + batchSize - 1) / batchSize) + .mapToObj( + i -> + paymentInsertEntities.subList( + i * batchSize, + Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) + .toList(); + batches.parallelStream().forEach(batch -> deleteAllInBatch(fdr, batch)); + + // FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); + log.info("End delete deleteByFdrAndPspId"); + + this.rePublish(fdr, pspId, fdrPublishEntity); + }) + .exceptionally( + e -> { + log.error("Exception during async saveOnStorage: ", e); + return null; + }); + } + + private void deleteAllInBatch(String fdr, List items) { + List indexes = items.stream().map(FdrPaymentInsertEntity::getIndex).toList(); + FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); } private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { @@ -362,13 +384,13 @@ private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEn } else { log.info("Starting add FdrInsertEntity in queue fdr message"); conversionQueue.addQueueFlowMessage( - FdrMessage.builder() - .fdr(fdrEntity.getFdr()) - .pspId(fdrEntity.getSender().getPspId()) - .organizationId(fdrEntity.getReceiver().getOrganizationId()) - .retry(0L) - .revision(fdrEntity.getRevision()) - .build()); + FdrMessage.builder() + .fdr(fdrEntity.getFdr()) + .pspId(fdrEntity.getSender().getPspId()) + .organizationId(fdrEntity.getReceiver().getOrganizationId()) + .retry(0L) + .revision(fdrEntity.getRevision()) + .build()); log.info("End add FdrInsertEntity in queue fdr message"); } } @@ -378,23 +400,22 @@ private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEnti String sessionId = (String) MDC.get(TRX_ID); MDC.put(EVENT_CATEGORY, EventTypeEnum.INTERNAL.name()); reService.sendEvent( - ReInternal.builder() - .serviceIdentifier(AppVersionEnum.FDR003) - .created(Instant.now()) - .sessionId(sessionId) - .eventType(EventTypeEnum.INTERNAL) - .fdrPhysicalDelete(false) - .fdrStatus(FdrStatusEnum.PUBLISHED) - .fdr(fdr) - .pspId(pspId) - .organizationId(fdrPublishEntity.getReceiver().getOrganizationId()) - .revision(fdrPublishEntity.getRevision()) - .fdrAction(FdrActionEnum.PUBLISH) - .build()); + ReInternal.builder() + .serviceIdentifier(AppVersionEnum.FDR003) + .created(Instant.now()) + .sessionId(sessionId) + .eventType(EventTypeEnum.INTERNAL) + .fdrPhysicalDelete(false) + .fdrStatus(FdrStatusEnum.PUBLISHED) + .fdr(fdr) + .pspId(pspId) + .organizationId(fdrPublishEntity.getReceiver().getOrganizationId()) + .revision(fdrPublishEntity.getRevision()) + .fdrAction(FdrActionEnum.PUBLISH) + .build()); log.info("End write to re"); } - @WithSpan(kind = SERVER) public void deleteByFdr(String action, String pspId, String fdr) { log.infof(AppMessageUtil.logExecute(action)); From 24b31bab7029b690e7bcd0191d9c0fe5506fc314 Mon Sep 17 00:00:00 2001 From: "angelo.caporaso" Date: Mon, 28 Oct 2024 10:38:49 +0100 Subject: [PATCH 25/41] [PAGOPA-2282] refactoring: Batch operations --- .../fdr/FdrPaymentPublishEntity.java | 5 +++ .../pagopa/fdr/service/psps/PspsService.java | 34 +++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java index 26e5026e..1c3558b5 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java @@ -6,6 +6,7 @@ import io.quarkus.mongodb.panache.common.MongoEntity; import io.quarkus.panache.common.Parameters; import io.quarkus.panache.common.Sort; +import io.smallrye.mutiny.Uni; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; import java.util.List; @@ -98,4 +99,8 @@ public static PanacheQuery findByPspAndIuvIur( } return find(query, sort, params); } + + public static void persistFdrPaymentPublishEntities(List fdrPaymentPublishEntities) { + persist(fdrPaymentPublishEntities); + } } diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 7f103a6f..68f61a55 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -320,7 +320,13 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); log.info("Starting persistent storage on Mongo of FDR payment entities"); - FdrPaymentPublishEntityReactive.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); + // FdrPaymentPublishEntityReactive.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); + int batchSizePub = 1000; + List> batchesPublish = IntStream + .range(0, (fdrPaymentPublishEntities.size() + batchSizePub - 1) / batchSizePub) + .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSizePub, Math.min((i + 1) * batchSizePub, fdrPaymentPublishEntities.size()))) + .toList(); + batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); log.info("End of persistent storage on Mongo of FDR payment entities"); // salva su storage dello storico @@ -349,17 +355,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern log.infof( "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); - int batchSize = 1000; - List> batches = - IntStream.range(0, (paymentInsertEntities.size() + batchSize - 1) / batchSize) - .mapToObj( - i -> - paymentInsertEntities.subList( - i * batchSize, - Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) - .toList(); - batches.parallelStream().forEach(batch -> deleteAllInBatch(fdr, batch)); - + deleteAllInBatch(fdr, paymentInsertEntities); // FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); log.info("End delete deleteByFdrAndPspId"); @@ -372,9 +368,17 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern }); } - private void deleteAllInBatch(String fdr, List items) { - List indexes = items.stream().map(FdrPaymentInsertEntity::getIndex).toList(); - FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); + private void deleteAllInBatch(String fdr, List paymentInsertEntities) { + int batchSize = 1000; + List> batches = IntStream + .range(0, (paymentInsertEntities.size() + batchSize - 1) / batchSize) + .mapToObj(i -> paymentInsertEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) + .toList(); + + batches.parallelStream().forEach(batch -> { + List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); + FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); + }); } private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { From 924ae47eed0e57f38876d5e75223d9e0ac7b617e Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Mon, 28 Oct 2024 09:55:52 +0000 Subject: [PATCH 26/41] Bump to version 1.0.21-10-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index ad937777..635ebc9d 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.37.0" -appVersion: "1.0.21-9-PAGOPA-2282" +version: "1.38.0" +appVersion: "1.0.21-10-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 98b4bd5f..665086c7 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-9-PAGOPA-2282 + tag: 1.0.21-10-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index ec5714dc..1e4390e2 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-9-PAGOPA-2282 + tag: 1.0.21-10-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index b41e27de..b3ac5d6a 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-9-PAGOPA-2282" + "version": "1.0.21-10-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index b6c5149d..4e116fe5 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-9-PAGOPA-2282" + "version": "1.0.21-10-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index e706f269..196719d5 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-9-PAGOPA-2282" + "version": "1.0.21-10-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 1fdb24b4..2b1d1e0f 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-9-PAGOPA-2282 + 1.0.21-10-PAGOPA-2282 3.11.0 1.18.26 From d452397af52ef86f0d14c11754ad4b25e3f8bca0 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:41:53 +0100 Subject: [PATCH 27/41] [PAGOPA-2282] refactoring: Sequential stream --- src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 68f61a55..c97ce2f8 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -375,7 +375,8 @@ private void deleteAllInBatch(String fdr, List paymentIn .mapToObj(i -> paymentInsertEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) .toList(); - batches.parallelStream().forEach(batch -> { + // sequential stream + batches.forEach(batch -> { List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); }); From 89bcc4e845294dedcb8d0f326ea7c2b875a4688a Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Mon, 28 Oct 2024 10:43:50 +0000 Subject: [PATCH 28/41] Bump to version 1.0.21-11-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 635ebc9d..4f09f5c7 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.38.0" -appVersion: "1.0.21-10-PAGOPA-2282" +version: "1.39.0" +appVersion: "1.0.21-11-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 665086c7..2e5d6894 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-10-PAGOPA-2282 + tag: 1.0.21-11-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 1e4390e2..4867ce24 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-10-PAGOPA-2282 + tag: 1.0.21-11-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index b3ac5d6a..4617f33b 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-10-PAGOPA-2282" + "version": "1.0.21-11-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 4e116fe5..1c74fb63 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-10-PAGOPA-2282" + "version": "1.0.21-11-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 196719d5..68ede76d 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-10-PAGOPA-2282" + "version": "1.0.21-11-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 2b1d1e0f..0bb460e7 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-10-PAGOPA-2282 + 1.0.21-11-PAGOPA-2282 3.11.0 1.18.26 From 5c8579df73ea0795d321da5c3392056eebb6ccac Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:28:56 +0100 Subject: [PATCH 29/41] [PAGOPA-2282] refactoring --- .../pagopa/fdr/service/psps/PspsService.java | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index c97ce2f8..07800779 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -303,8 +303,6 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrEntity.getComputedSumPayments()); } - log.debug("FdrInsertEntity PUBLISHED"); - log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, pspId); List paymentInsertEntities = FdrPaymentInsertEntity.findByFdrAndPspId(fdr, pspId) @@ -316,49 +314,33 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.setUpdated(now); fdrPublishEntity.setPublished(now); fdrPublishEntity.setStatus(FdrStatusEnumEntity.PUBLISHED); - List fdrPaymentPublishEntities = - mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); + List fdrPaymentPublishEntities = mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); - log.info("Starting persistent storage on Mongo of FDR payment entities"); - // FdrPaymentPublishEntityReactive.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); - int batchSizePub = 1000; - List> batchesPublish = IntStream - .range(0, (fdrPaymentPublishEntities.size() + batchSizePub - 1) / batchSizePub) - .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSizePub, Math.min((i + 1) * batchSizePub, fdrPaymentPublishEntities.size()))) - .toList(); - batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); - log.info("End of persistent storage on Mongo of FDR payment entities"); + // writes in fdr_payment_publish collection + this.parallelPersist(fdr, fdrPaymentPublishEntities); - // salva su storage dello storico + // save as JSON file in history storage HistoryBlobBody body = historyService.saveJsonFile(fdrPublishEntity, fdrPaymentPublishEntities); fdrPublishEntity.setRefJson(body); fdrPublishEntity.persistEntity(); - log.info("Starting saveOnStorage storage on BlobStorage of FDR payment entities"); CompletableFuture executeSaveOnStorage = CompletableFuture.supplyAsync( () -> { historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); - log.info("Async saveOnStorage successful executed"); return true; }); executeSaveOnStorage .thenAccept( value -> { - log.info("End of saveOnStorage storage on BlobStorage of FDR payment entities"); - + // queue this.addToConversionQueue(internalPublish, fdrEntity); - - // todo: handles deletes, check insert overwrites + // delete fdrEntity.delete(); - log.infof( - "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", - fdrEntity.getRevision(), fdr, pspId); - deleteAllInBatch(fdr, paymentInsertEntities); - // FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); - log.info("End delete deleteByFdrAndPspId"); - + this.batchDelete(fdr, paymentInsertEntities); + log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, pspId); + // re this.rePublish(fdr, pspId, fdrPublishEntity); }) .exceptionally( @@ -368,13 +350,12 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern }); } - private void deleteAllInBatch(String fdr, List paymentInsertEntities) { + private void batchDelete(String fdr, List paymentInsertEntities) { int batchSize = 1000; List> batches = IntStream .range(0, (paymentInsertEntities.size() + batchSize - 1) / batchSize) .mapToObj(i -> paymentInsertEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) .toList(); - // sequential stream batches.forEach(batch -> { List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); @@ -382,6 +363,16 @@ private void deleteAllInBatch(String fdr, List paymentIn }); } + private void parallelPersist(String fdr, List fdrPaymentPublishEntities) { + int batchSize = 1000; + List> batchesPublish = IntStream + .range(0, (fdrPaymentPublishEntities.size() + batchSize - 1) / batchSize) + .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, fdrPaymentPublishEntities.size()))) + .toList(); + batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); + log.infof("Published fdrPaymentPublishEntities of fdr[%s]", fdr); + } + private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { // add to conversion queue if (internalPublish) { From f958264689a1d221551e76ed1645863c94120af6 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Mon, 28 Oct 2024 13:31:05 +0000 Subject: [PATCH 30/41] Bump to version 1.0.21-12-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 4f09f5c7..e25e91ba 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.39.0" -appVersion: "1.0.21-11-PAGOPA-2282" +version: "1.40.0" +appVersion: "1.0.21-12-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 2e5d6894..8115ffb8 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-11-PAGOPA-2282 + tag: 1.0.21-12-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 4867ce24..96067e57 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-11-PAGOPA-2282 + tag: 1.0.21-12-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 4617f33b..55b1d413 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-11-PAGOPA-2282" + "version": "1.0.21-12-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 1c74fb63..28a38268 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-11-PAGOPA-2282" + "version": "1.0.21-12-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 68ede76d..58ca1dad 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-11-PAGOPA-2282" + "version": "1.0.21-12-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 0bb460e7..403c7bc3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-11-PAGOPA-2282 + 1.0.21-12-PAGOPA-2282 3.11.0 1.18.26 From e00d7db7ebac0b2daf1d0bc941f632ee17834e31 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:49:12 +0100 Subject: [PATCH 31/41] [PAGOPA-2282] test: Add py test --- performance-test/python-test/main.py | 44 ++++++++++ performance-test/python-test/methods.py | 106 ++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 performance-test/python-test/main.py create mode 100644 performance-test/python-test/methods.py diff --git a/performance-test/python-test/main.py b/performance-test/python-test/main.py new file mode 100644 index 00000000..5828067e --- /dev/null +++ b/performance-test/python-test/main.py @@ -0,0 +1,44 @@ +import sys +import logging, time +import methods + + +NUMBER_OF_PAYMENTS = 5000 +MAX_PAYMENTS_PER_ADD_OPERATION = 1000 + +def main(URL, subkey): + logging.basicConfig(level=logging.INFO) + + flow_date = "2024-10-15" + tmstmp = timestamp = int(time.time()) + flow_name = f"{flow_date}88888888888-{tmstmp}" + + create_url = URL + f"/psps/88888888888/fdrs/{flow_name}" + methods.create_empty_flow(create_url, flow_name, flow_date, NUMBER_OF_PAYMENTS, subkey) + + + add_url = URL + f"/psps/88888888888/fdrs/{flow_name}/payments/add" + methods.add_payments(add_url, NUMBER_OF_PAYMENTS, MAX_PAYMENTS_PER_ADD_OPERATION, flow_date, subkey) + + + publish_url = URL + f"/psps/88888888888/fdrs/{flow_name}/publish" + methods.publish_payments(publish_url, subkey) + +def get_url(env): + if env == 'dev': + return "https://api.dev.platform.pagopa.it/fdr-psp/service/v1" + elif env == 'uat': + return "https://upload.uat.platform.pagopa.it/fdr-psp/service/v1" + else: + raise ValueError(f"Invalid environment: {env}. Please use 'dev' or 'uat'.") + +if __name__ == "__main__": + try: + env = sys.argv[1] + key = sys.argv[2] + url = get_url(env) + main(url, key) + except IndexError: + print("Usage: python3 main.py \ni.e. python3 main.py dev your-key") + except ValueError as e: + print(e) \ No newline at end of file diff --git a/performance-test/python-test/methods.py b/performance-test/python-test/methods.py new file mode 100644 index 00000000..4211c062 --- /dev/null +++ b/performance-test/python-test/methods.py @@ -0,0 +1,106 @@ +import json, logging, requests, time +import utility + + +def create_empty_flow(url, flow_name, flow_date, total_payments, key): + + headers = { + "Ocp-Apim-Subscription-Key": key + } + request = { + "fdr": flow_name, + "fdrDate": f"{flow_date}T12:00:00.000Z", + "sender": { + "type": "LEGAL_PERSON", + "id": "SELBIT2B", + "pspId": "88888888888", + "pspName": "Bank", + "pspBrokerId": "88888888888", + "channelId": "88888888888_01", + "password": "PLACEHOLDER" + }, + "receiver": { + "id": "APPBIT2B", + "organizationId": "15376371009", + "organizationName": "PagoPA" + }, + "regulation": "SEPA - Bonifico xzy", + "regulationDate": f"{flow_date}T12:00:00.000Z", + "bicCodePouringBank": "UNCRITMMXXX", + "totPayments": total_payments, + "sumPayments": total_payments * 10 + } + logging.info(f"\nSend request to: [{url}]\nRequest: [{request}]") + start_time = time.time() + response = requests.post(url=url, + data=json.dumps(request), + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + + +def add_payments(url, total_payments, max_per_add, date, key): + + headers = { + "Ocp-Apim-Subscription-Key": key + } + + total_amount = total_payments * 10 + payments = generate_payments(total_payments, total_amount, date) + request = [] + + total_requests = int(total_payments / max_per_add) + for req_idx in range(total_requests): + extracted_payments = { + "payments": payments[(req_idx * max_per_add):(req_idx * max_per_add + max_per_add)] + } + request = json.dumps(extracted_payments) + logging.info(f"\nSend request to: [{url}]\nRequest: [TOO LONG]") + start_time = time.time() + response = requests.put(url=url, + data=request, + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + +def publish_payments(url, key): + headers = { + "Ocp-Apim-Subscription-Key": key + } + logging.info(f"\nSend request to: [{url}]") + start_time = time.time() + response = requests.post(url=url, + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + +######### +# UTILS # +######### +def generate_payments(number_of_payments, total_amount, date): + + payments = [] + iuvs = set() + for idx in range(number_of_payments): + iuv = utility.get_random_numeric_string(15) + if iuv not in iuvs: + iuvs.add(iuv) + payments.append({ + "index": idx + 1, + "iuv": iuv, + "iur": utility.get_random_numeric_string(11), + "pay": total_amount / number_of_payments, + "idTransfer": 1, + "payStatus": "EXECUTED", + "payDate": f"{date}T12:00:00.000Z" + }) + else: + idx -= 1 + return payments + +def get_random_numeric_string(size, dataset = "0123456789"): + random_string = ''.join(random.choices(dataset, k=size)) + return random_string \ No newline at end of file From 01a8a7fc5243d21b844f2b8fdcc7a4083b04fb0c Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:55:39 +0100 Subject: [PATCH 32/41] Fix code scanning alert no. 727: Log Injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../java/it/gov/pagopa/fdr/service/psps/PspsService.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 07800779..77c006f4 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -273,6 +273,10 @@ public void deletePayment( } @WithSpan(kind = SERVER) + private String sanitize(String input) { + return input.replaceAll("[\\n\\r\\t]", "_"); + } + public void publishByFdr(String action, String pspId, String fdr, boolean internalPublish) { log.infof(AppMessageUtil.logExecute(action)); @@ -303,7 +307,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrEntity.getComputedSumPayments()); } - log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, pspId); + log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, sanitize(pspId)); List paymentInsertEntities = FdrPaymentInsertEntity.findByFdrAndPspId(fdr, pspId) .project(FdrPaymentInsertEntity.class) @@ -339,7 +343,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern // delete fdrEntity.delete(); this.batchDelete(fdr, paymentInsertEntities); - log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, pspId); + log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, sanitize(pspId)); // re this.rePublish(fdr, pspId, fdrPublishEntity); }) From 2178f5de5263537e9aeb9a300f46358ccab79bf4 Mon Sep 17 00:00:00 2001 From: Jacopo Carlini Date: Mon, 28 Oct 2024 18:02:43 +0100 Subject: [PATCH 33/41] Fix code scanning alert no. 729: Log Injection Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 77c006f4..6e61f75d 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -307,7 +307,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrEntity.getComputedSumPayments()); } - log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, sanitize(pspId)); + log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", sanitize(fdr), sanitize(pspId)); List paymentInsertEntities = FdrPaymentInsertEntity.findByFdrAndPspId(fdr, pspId) .project(FdrPaymentInsertEntity.class) From 180faf532cf3ddbaa8953f6e005779b5177dfd40 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:43:12 +0100 Subject: [PATCH 34/41] [PAGOPA-2282] fix --- performance-test/python-test/main.py | 4 ++-- performance-test/python-test/methods.py | 6 +++--- .../fdr/FdrPaymentInsertEntity.java | 6 ++++++ .../fdr/FdrPaymentPublishEntityReactive.java | 19 ------------------- .../pagopa/fdr/service/psps/PspsService.java | 15 +++++++-------- src/main/resources/application.properties | 5 ----- 6 files changed, 18 insertions(+), 37 deletions(-) delete mode 100644 src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java diff --git a/performance-test/python-test/main.py b/performance-test/python-test/main.py index 5828067e..ee087c97 100644 --- a/performance-test/python-test/main.py +++ b/performance-test/python-test/main.py @@ -3,13 +3,13 @@ import methods -NUMBER_OF_PAYMENTS = 5000 +NUMBER_OF_PAYMENTS = 30000 MAX_PAYMENTS_PER_ADD_OPERATION = 1000 def main(URL, subkey): logging.basicConfig(level=logging.INFO) - flow_date = "2024-10-15" + flow_date = "2024-10-30" tmstmp = timestamp = int(time.time()) flow_name = f"{flow_date}88888888888-{tmstmp}" diff --git a/performance-test/python-test/methods.py b/performance-test/python-test/methods.py index 4211c062..3045161e 100644 --- a/performance-test/python-test/methods.py +++ b/performance-test/python-test/methods.py @@ -1,5 +1,5 @@ +import random import json, logging, requests, time -import utility def create_empty_flow(url, flow_name, flow_date, total_payments, key): @@ -85,13 +85,13 @@ def generate_payments(number_of_payments, total_amount, date): payments = [] iuvs = set() for idx in range(number_of_payments): - iuv = utility.get_random_numeric_string(15) + iuv = get_random_numeric_string(15) if iuv not in iuvs: iuvs.add(iuv) payments.append({ "index": idx + 1, "iuv": iuv, - "iur": utility.get_random_numeric_string(11), + "iur": get_random_numeric_string(11), "pay": total_amount / number_of_payments, "idTransfer": 1, "payStatus": "EXECUTED", diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java index f9fdbecf..fffc5c97 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java @@ -67,6 +67,12 @@ public static PanacheQuery findByFdrAndPspId(String fdr, Parameters.with("fdr", fdr).and("pspId", pspId).map()); } + public static void deleteByFdrAndIds(String fdr, List objectIds) { + delete( + "ref_fdr = :fdr and _id in :objectIds", + Parameters.with("fdr", fdr).and("ids", objectIds).map()); + } + public static long deleteByFdrAndIndexes(String fdr, List indexList) { return delete( "ref_fdr = :fdr and index in :indexes", diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java deleted file mode 100644 index ddf17536..00000000 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntityReactive.java +++ /dev/null @@ -1,19 +0,0 @@ -package it.gov.pagopa.fdr.repository.fdr; - -import io.quarkus.mongodb.panache.common.MongoEntity; -import io.quarkus.mongodb.panache.reactive.ReactivePanacheMongoEntity; -import io.smallrye.mutiny.Uni; -import java.util.List; -import lombok.Data; -import lombok.EqualsAndHashCode; - -@Data -@EqualsAndHashCode(callSuper = true) -@MongoEntity(collection = "fdr_payment_publish") -public class FdrPaymentPublishEntityReactive extends ReactivePanacheMongoEntity { - - // Reactive persistence method for FdrPaymentPublishEntity objects - public static Uni persistFdrPaymentPublishEntities(List entities) { - return persist(entities); - } -} diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 6e61f75d..26ab1cee 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -35,6 +35,7 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.bson.types.ObjectId; import org.jboss.logging.Logger; import org.jboss.logging.MDC; @@ -362,8 +363,8 @@ private void batchDelete(String fdr, List paymentInsertE .toList(); // sequential stream batches.forEach(batch -> { - List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); - FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); + List objectIds = paymentInsertEntities.stream().map(entity -> entity.id).toList(); + FdrPaymentInsertEntity.deleteByFdrAndIds(fdr, objectIds); }); } @@ -374,15 +375,15 @@ private void parallelPersist(String fdr, List fdrPaymen .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, fdrPaymentPublishEntities.size()))) .toList(); batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); - log.infof("Published fdrPaymentPublishEntities of fdr[%s]", fdr); + log.debugf("Published fdrPaymentPublishEntities of fdr[%s]", fdr); } private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { // add to conversion queue if (internalPublish) { - log.info("NOT Add FdrInsertEntity in queue fdr message"); + log.debugf("NOT Add FdrInsertEntity in queue fdr message"); } else { - log.info("Starting add FdrInsertEntity in queue fdr message"); + log.debugf("Starting add FdrInsertEntity in queue fdr message"); conversionQueue.addQueueFlowMessage( FdrMessage.builder() .fdr(fdrEntity.getFdr()) @@ -391,12 +392,11 @@ private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEn .retry(0L) .revision(fdrEntity.getRevision()) .build()); - log.info("End add FdrInsertEntity in queue fdr message"); + log.debugf("End add FdrInsertEntity in queue fdr message"); } } private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEntity) { - log.info("Starting write to re"); String sessionId = (String) MDC.get(TRX_ID); MDC.put(EVENT_CATEGORY, EventTypeEnum.INTERNAL.name()); reService.sendEvent( @@ -413,7 +413,6 @@ private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEnti .revision(fdrPublishEntity.getRevision()) .fdrAction(FdrActionEnum.PUBLISH) .build()); - log.info("End write to re"); } @WithSpan(kind = SERVER) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index da9a9a02..83107936 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -69,11 +69,6 @@ quarkus.mongodb.connection-string=${MONGODB_CONNECTION_STRING:${mockserver.mongo %dev.quarkus.mongodb.connection-string=mongodb://root:example@localhost:27017/ quarkus.mongodb.database=fdr -################### -## Vert.x -################### -# quarkus.vertx.max-event-loop-execute-time=2s - ################### ## ADAPTER API CONFIG CACHE ################### From 299f981d91add3beaf8f5c94fece3c45efb119f9 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Wed, 30 Oct 2024 09:45:42 +0000 Subject: [PATCH 35/41] Bump to version 1.0.21-13-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index e25e91ba..8ccf83ac 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.40.0" -appVersion: "1.0.21-12-PAGOPA-2282" +version: "1.41.0" +appVersion: "1.0.21-13-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index 8115ffb8..d9f388bb 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-12-PAGOPA-2282 + tag: 1.0.21-13-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 96067e57..e958729b 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-12-PAGOPA-2282 + tag: 1.0.21-13-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 55b1d413..9666c26a 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-12-PAGOPA-2282" + "version": "1.0.21-13-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index 28a38268..e2eda86b 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-12-PAGOPA-2282" + "version": "1.0.21-13-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 58ca1dad..7116a85a 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-12-PAGOPA-2282" + "version": "1.0.21-13-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 403c7bc3..8dfeb8d9 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-12-PAGOPA-2282 + 1.0.21-13-PAGOPA-2282 3.11.0 1.18.26 From 6d4fa5a89cd1a658e2fea837627586ad5ec7e9ae Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:28:10 +0100 Subject: [PATCH 36/41] [PAGOPA-2282] fix --- src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 26ab1cee..0157b9bc 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -273,7 +273,6 @@ public void deletePayment( .build()); } - @WithSpan(kind = SERVER) private String sanitize(String input) { return input.replaceAll("[\\n\\r\\t]", "_"); } From 581d965064beae6fe5a20702f1fd4ec98f1bb23e Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Wed, 30 Oct 2024 10:29:24 +0000 Subject: [PATCH 37/41] Bump to version 1.0.21-14-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 8ccf83ac..70593a83 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.41.0" -appVersion: "1.0.21-13-PAGOPA-2282" +version: "1.42.0" +appVersion: "1.0.21-14-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index d9f388bb..de2210e5 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-13-PAGOPA-2282 + tag: 1.0.21-14-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index e958729b..5906e1a6 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-13-PAGOPA-2282 + tag: 1.0.21-14-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 9666c26a..59edea5f 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-13-PAGOPA-2282" + "version": "1.0.21-14-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index e2eda86b..e93838fe 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-13-PAGOPA-2282" + "version": "1.0.21-14-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index 7116a85a..a75adf9c 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-13-PAGOPA-2282" + "version": "1.0.21-14-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index 8dfeb8d9..7622e3f8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-13-PAGOPA-2282 + 1.0.21-14-PAGOPA-2282 3.11.0 1.18.26 From 0395bdea7854e354e61586386f831999303207b3 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:08:43 +0100 Subject: [PATCH 38/41] [PAGOPA-2282] fix(delete-by-index): Add smallrye-fault-tolerance --- README.md | 5 +++++ performance-test/python-test/main.py | 4 ++-- pom.xml | 4 ++++ .../fdr/repository/fdr/FdrPaymentInsertEntity.java | 10 ++++------ .../it/gov/pagopa/fdr/service/psps/PspsService.java | 6 +++--- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index c4128509..37f9a7ac 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,11 @@ You can run your application in dev mode that enables live coding using: ```shell script ./mvnw compile quarkus:dev ``` +Otherwise, with quarkus CLI: +``` +brew install quarkusio/tap/quarkus +quarkus dev -DskipTests=true +``` > **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only > at http://localhost:8080/q/dev/. diff --git a/performance-test/python-test/main.py b/performance-test/python-test/main.py index ee087c97..4929a35f 100644 --- a/performance-test/python-test/main.py +++ b/performance-test/python-test/main.py @@ -3,8 +3,8 @@ import methods -NUMBER_OF_PAYMENTS = 30000 -MAX_PAYMENTS_PER_ADD_OPERATION = 1000 +NUMBER_OF_PAYMENTS = 300 +MAX_PAYMENTS_PER_ADD_OPERATION = 100 def main(URL, subkey): logging.basicConfig(level=logging.INFO) diff --git a/pom.xml b/pom.xml index 7622e3f8..cb8408c5 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,10 @@ 1.0.0 provided + + io.quarkus + quarkus-smallrye-fault-tolerance + org.testcontainers junit-jupiter diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java index fffc5c97..16e581d4 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java @@ -8,11 +8,13 @@ import io.quarkus.panache.common.Sort; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; import org.bson.codecs.pojo.annotations.BsonProperty; import org.bson.types.ObjectId; +import org.eclipse.microprofile.faulttolerance.Retry; @Data @EqualsAndHashCode(callSuper = true) @@ -67,12 +69,8 @@ public static PanacheQuery findByFdrAndPspId(String fdr, Parameters.with("fdr", fdr).and("pspId", pspId).map()); } - public static void deleteByFdrAndIds(String fdr, List objectIds) { - delete( - "ref_fdr = :fdr and _id in :objectIds", - Parameters.with("fdr", fdr).and("ids", objectIds).map()); - } - + // https://quarkus.io/guides/smallrye-fault-tolerance + @Retry(delay = 500, delayUnit = ChronoUnit.MILLIS) public static long deleteByFdrAndIndexes(String fdr, List indexList) { return delete( "ref_fdr = :fdr and index in :indexes", diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 0157b9bc..44838ff9 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -341,8 +341,8 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern // queue this.addToConversionQueue(internalPublish, fdrEntity); // delete - fdrEntity.delete(); this.batchDelete(fdr, paymentInsertEntities); + fdrEntity.delete(); log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, sanitize(pspId)); // re this.rePublish(fdr, pspId, fdrPublishEntity); @@ -362,8 +362,8 @@ private void batchDelete(String fdr, List paymentInsertE .toList(); // sequential stream batches.forEach(batch -> { - List objectIds = paymentInsertEntities.stream().map(entity -> entity.id).toList(); - FdrPaymentInsertEntity.deleteByFdrAndIds(fdr, objectIds); + List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); + FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); }); } From a0fe020c07267673398f976b1a2c1dba39dfad33 Mon Sep 17 00:00:00 2001 From: pagopa-github-bot Date: Wed, 30 Oct 2024 14:10:09 +0000 Subject: [PATCH 39/41] Bump to version 1.0.21-15-PAGOPA-2282 [skip ci] --- helm/Chart.yaml | 4 ++-- helm/values-dev.yaml | 2 +- helm/values-uat.yaml | 2 +- openapi/openapi_internal.json | 2 +- openapi/openapi_organization.json | 2 +- openapi/openapi_psp.json | 2 +- pom.xml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 70593a83..a9e5e2e0 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.42.0" -appVersion: "1.0.21-14-PAGOPA-2282" +version: "1.43.0" +appVersion: "1.0.21-15-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index de2210e5..e93abf01 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-14-PAGOPA-2282 + tag: 1.0.21-15-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 5906e1a6..aa458dee 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.21-14-PAGOPA-2282 + tag: 1.0.21-15-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index 59edea5f..523e3cab 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-14-PAGOPA-2282" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index e93838fe..e5cbc075 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-14-PAGOPA-2282" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index a75adf9c..7a3389bc 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.21-14-PAGOPA-2282" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/pom.xml b/pom.xml index cb8408c5..19683857 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.21-14-PAGOPA-2282 + 1.0.21-15-PAGOPA-2282 3.11.0 1.18.26 From c419bf645b9f591003952c17b06baacd975b0827 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:13:38 +0100 Subject: [PATCH 40/41] [PAGOPA-2282] fix(ingress): Rollback timeout --- helm/values-uat.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index aa458dee..a336c767 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -97,9 +97,9 @@ microservice-chart: path: /pagopa-fdr-service(/|$)(.*) # currently set ingress timeout to 50m, until async handling will be introduced annotations: - nginx.ingress.kubernetes.io/proxy-connect-timeout: "3000" - nginx.ingress.kubernetes.io/proxy-read-timeout: "3000" - nginx.ingress.kubernetes.io/proxy-send-timeout: "3000" - nginx.ingress.kubernetes.io/send_timeout: "3000" + nginx.ingress.kubernetes.io/proxy-connect-timeout: "300" + nginx.ingress.kubernetes.io/proxy-read-timeout: "300" + nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + nginx.ingress.kubernetes.io/send_timeout: "300" canaryDelivery: create: false From bbe74918c4d3410ef5163e432b5a9c802ffe8700 Mon Sep 17 00:00:00 2001 From: Angelo Caporaso <56113767+cap-ang@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:25:26 +0100 Subject: [PATCH 41/41] [PAGOPA-2282] refactoring(utils): Add sanitize --- .../it/gov/pagopa/fdr/service/psps/PspsService.java | 12 +++++------- src/main/java/it/gov/pagopa/fdr/util/StringUtil.java | 7 +++++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index 44838ff9..044cd462 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -15,6 +15,7 @@ import it.gov.pagopa.fdr.repository.fdr.projection.FdrInsertProjection; import it.gov.pagopa.fdr.repository.fdr.projection.FdrPublishByPspProjection; import it.gov.pagopa.fdr.repository.fdr.projection.FdrPublishRevisionProjection; +import it.gov.pagopa.fdr.rest.validation.CommonValidationService; import it.gov.pagopa.fdr.service.conversion.ConversionService; import it.gov.pagopa.fdr.service.conversion.message.FdrMessage; import it.gov.pagopa.fdr.service.dto.*; @@ -25,6 +26,7 @@ import it.gov.pagopa.fdr.service.re.model.*; import it.gov.pagopa.fdr.util.AppDBUtil; import it.gov.pagopa.fdr.util.AppMessageUtil; +import it.gov.pagopa.fdr.util.StringUtil; import jakarta.enterprise.context.ApplicationScoped; import java.math.BigDecimal; import java.time.Instant; @@ -273,10 +275,6 @@ public void deletePayment( .build()); } - private String sanitize(String input) { - return input.replaceAll("[\\n\\r\\t]", "_"); - } - public void publishByFdr(String action, String pspId, String fdr, boolean internalPublish) { log.infof(AppMessageUtil.logExecute(action)); @@ -307,7 +305,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrEntity.getComputedSumPayments()); } - log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", sanitize(fdr), sanitize(pspId)); + log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", StringUtil.sanitize(fdr), StringUtil.sanitize(pspId)); List paymentInsertEntities = FdrPaymentInsertEntity.findByFdrAndPspId(fdr, pspId) .project(FdrPaymentInsertEntity.class) @@ -343,7 +341,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern // delete this.batchDelete(fdr, paymentInsertEntities); fdrEntity.delete(); - log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, sanitize(pspId)); + log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", StringUtil.sanitize(fdr), StringUtil.sanitize(pspId)); // re this.rePublish(fdr, pspId, fdrPublishEntity); }) @@ -374,7 +372,7 @@ private void parallelPersist(String fdr, List fdrPaymen .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, fdrPaymentPublishEntities.size()))) .toList(); batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); - log.debugf("Published fdrPaymentPublishEntities of fdr[%s]", fdr); + log.debugf("Published fdrPaymentPublishEntities of fdr[%s]", StringUtil.sanitize(fdr)); } private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { diff --git a/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java b/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java index 519e64c0..b624bfb9 100644 --- a/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java +++ b/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java @@ -17,4 +17,11 @@ public static byte[] zip(String str) throws IOException { bais.close(); return compressed; } + + // Replace newline, carriage return, tab, single quote, double quote, and backslash characters + public static String sanitize(String input) { + if (input == null) + return null; + return input.replaceAll("[\\n\\r\\t'\"\\\\]", "_"); + } }