diff --git a/.github/workflows/04_release_and_deploy.yml b/.github/workflows/04_release_and_deploy.yml index ad863646..9b3932df 100644 --- a/.github/workflows/04_release_and_deploy.yml +++ b/.github/workflows/04_release_and_deploy.yml @@ -18,15 +18,13 @@ on: - dev - uat - prod - semver: + version: required: false type: choice description: Select the version options: - '' - - skip - - promote - - patch + - skip_or_promote - new_release - breaking_change beta: @@ -40,10 +38,6 @@ on: environment: required: true type: string - semver: - required: true - type: string - default: skip permissions: packages: write @@ -59,14 +53,47 @@ jobs: name: Setup runs-on: ubuntu-latest outputs: - semver: ${{ steps.semver_setup.outputs.semver }} - environment: ${{ steps.semver_setup.outputs.environment }} + semver: ${{ steps.get_semver.outputs.semver }} + environment: ${{ steps.get_env.outputs.environment }} steps: - - name: Semver setup - id: semver_setup - uses: pagopa/github-actions-template/nodo5-semver-setup@ce252c8501c9242bd6045f7cdd650736b2f38777 - with: - semver: ${{ inputs.semver }} + - name: pull request rejected + if: github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged != true + run: | + echo "❌ PR was closed without a merge" + exit 1 + + # Set Semvar + - run: echo "SEMVER=patch" >> $GITHUB_ENV + + - if: ${{ (github.event.pull_request.merged && contains(github.event.pull_request.labels.*.name, 'breaking-change')) }} + run: echo "SEMVER=major" >> $GITHUB_ENV + + # force semver if dev, !=main or skip release + - if: ${{ inputs.version == 'new_release' }} + run: echo "SEMVER=minor" >> $GITHUB_ENV + + - if: ${{ inputs.version == 'breaking_change' }} + run: echo "SEMVER=major" >> $GITHUB_ENV + + - if: ${{ github.ref_name != 'main' }} + run: echo "SEMVER=buildNumber" >> $GITHUB_ENV + + - if: ${{ inputs.version == 'skip_or_promote' }} + run: echo "SEMVER=skip" >> $GITHUB_ENV + + - id: get_semver + name: Set Output + run: echo "semver=${{env.SEMVER}}" >> $GITHUB_OUTPUT + + # Set Environment + - run: echo "ENVIRNOMENT=${{ inputs.environment}}" >> $GITHUB_ENV + + - if: ${{ inputs.environment == null }} + run: echo "ENVIRNOMENT=dev" >> $GITHUB_ENV + + - id: get_env + name: Set Output + run: echo "environment=${{env.ENVIRNOMENT}}" >> $GITHUB_OUTPUT release: needs: [setup] diff --git a/README.md b/README.md index c4128509..37f9a7ac 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,11 @@ You can run your application in dev mode that enables live coding using: ```shell script ./mvnw compile quarkus:dev ``` +Otherwise, with quarkus CLI: +``` +brew install quarkusio/tap/quarkus +quarkus dev -DskipTests=true +``` > **_NOTE:_** Quarkus now ships with a Dev UI, which is available in dev mode only > at http://localhost:8080/q/dev/. diff --git a/helm/Chart.yaml b/helm/Chart.yaml index ea6906bc..a9e5e2e0 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: pagopa-fdr-chart description: Flussi di rendicontazioni type: application -version: "1.30.0" -appVersion: "1.0.23" +version: "1.43.0" +appVersion: "1.0.21-15-PAGOPA-2282" dependencies: - name: microservice-chart version: 3.0.0 diff --git a/helm/values-dev.yaml b/helm/values-dev.yaml index ec0e9219..e93abf01 100644 --- a/helm/values-dev.yaml +++ b/helm/values-dev.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.23 + tag: 1.0.21-15-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/helm/values-uat.yaml b/helm/values-uat.yaml index 6bcea45d..aa458dee 100644 --- a/helm/values-uat.yaml +++ b/helm/values-uat.yaml @@ -4,7 +4,7 @@ microservice-chart: fullnameOverride: "" image: repository: ghcr.io/pagopa/pagopa-fdr - tag: 1.0.23 + tag: 1.0.21-15-PAGOPA-2282 pullPolicy: Always readinessProbe: httpGet: diff --git a/openapi/openapi_internal.json b/openapi/openapi_internal.json index c159d5b3..523e3cab 100644 --- a/openapi/openapi_internal.json +++ b/openapi/openapi_internal.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.23" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_organization.json b/openapi/openapi_organization.json index a0cb1088..e5cbc075 100644 --- a/openapi/openapi_organization.json +++ b/openapi/openapi_organization.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.23" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/openapi/openapi_psp.json b/openapi/openapi_psp.json index e9f42024..7a3389bc 100644 --- a/openapi/openapi_psp.json +++ b/openapi/openapi_psp.json @@ -4,7 +4,7 @@ "title": "FDR - Flussi di rendicontazione (local)", "description": "Manage FDR ( aka \"Flussi di Rendicontazione\" ) exchanged between PSP and EC", "termsOfService": "https://www.pagopa.gov.it/", - "version": "1.0.23" + "version": "1.0.21-15-PAGOPA-2282" }, "servers": [ { diff --git a/performance-test/python-test/main.py b/performance-test/python-test/main.py new file mode 100644 index 00000000..4929a35f --- /dev/null +++ b/performance-test/python-test/main.py @@ -0,0 +1,44 @@ +import sys +import logging, time +import methods + + +NUMBER_OF_PAYMENTS = 300 +MAX_PAYMENTS_PER_ADD_OPERATION = 100 + +def main(URL, subkey): + logging.basicConfig(level=logging.INFO) + + flow_date = "2024-10-30" + tmstmp = timestamp = int(time.time()) + flow_name = f"{flow_date}88888888888-{tmstmp}" + + create_url = URL + f"/psps/88888888888/fdrs/{flow_name}" + methods.create_empty_flow(create_url, flow_name, flow_date, NUMBER_OF_PAYMENTS, subkey) + + + add_url = URL + f"/psps/88888888888/fdrs/{flow_name}/payments/add" + methods.add_payments(add_url, NUMBER_OF_PAYMENTS, MAX_PAYMENTS_PER_ADD_OPERATION, flow_date, subkey) + + + publish_url = URL + f"/psps/88888888888/fdrs/{flow_name}/publish" + methods.publish_payments(publish_url, subkey) + +def get_url(env): + if env == 'dev': + return "https://api.dev.platform.pagopa.it/fdr-psp/service/v1" + elif env == 'uat': + return "https://upload.uat.platform.pagopa.it/fdr-psp/service/v1" + else: + raise ValueError(f"Invalid environment: {env}. Please use 'dev' or 'uat'.") + +if __name__ == "__main__": + try: + env = sys.argv[1] + key = sys.argv[2] + url = get_url(env) + main(url, key) + except IndexError: + print("Usage: python3 main.py \ni.e. python3 main.py dev your-key") + except ValueError as e: + print(e) \ No newline at end of file diff --git a/performance-test/python-test/methods.py b/performance-test/python-test/methods.py new file mode 100644 index 00000000..3045161e --- /dev/null +++ b/performance-test/python-test/methods.py @@ -0,0 +1,106 @@ +import random +import json, logging, requests, time + + +def create_empty_flow(url, flow_name, flow_date, total_payments, key): + + headers = { + "Ocp-Apim-Subscription-Key": key + } + request = { + "fdr": flow_name, + "fdrDate": f"{flow_date}T12:00:00.000Z", + "sender": { + "type": "LEGAL_PERSON", + "id": "SELBIT2B", + "pspId": "88888888888", + "pspName": "Bank", + "pspBrokerId": "88888888888", + "channelId": "88888888888_01", + "password": "PLACEHOLDER" + }, + "receiver": { + "id": "APPBIT2B", + "organizationId": "15376371009", + "organizationName": "PagoPA" + }, + "regulation": "SEPA - Bonifico xzy", + "regulationDate": f"{flow_date}T12:00:00.000Z", + "bicCodePouringBank": "UNCRITMMXXX", + "totPayments": total_payments, + "sumPayments": total_payments * 10 + } + logging.info(f"\nSend request to: [{url}]\nRequest: [{request}]") + start_time = time.time() + response = requests.post(url=url, + data=json.dumps(request), + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + + +def add_payments(url, total_payments, max_per_add, date, key): + + headers = { + "Ocp-Apim-Subscription-Key": key + } + + total_amount = total_payments * 10 + payments = generate_payments(total_payments, total_amount, date) + request = [] + + total_requests = int(total_payments / max_per_add) + for req_idx in range(total_requests): + extracted_payments = { + "payments": payments[(req_idx * max_per_add):(req_idx * max_per_add + max_per_add)] + } + request = json.dumps(extracted_payments) + logging.info(f"\nSend request to: [{url}]\nRequest: [TOO LONG]") + start_time = time.time() + response = requests.put(url=url, + data=request, + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + +def publish_payments(url, key): + headers = { + "Ocp-Apim-Subscription-Key": key + } + logging.info(f"\nSend request to: [{url}]") + start_time = time.time() + response = requests.post(url=url, + headers=headers) + end_time = time.time() + logging.info(f"=====\nElapsed time: [{(end_time - start_time):.3f} sec] Status Code: [{response.status_code}]\nResponse: {response.json()}\n==================") + + +######### +# UTILS # +######### +def generate_payments(number_of_payments, total_amount, date): + + payments = [] + iuvs = set() + for idx in range(number_of_payments): + iuv = get_random_numeric_string(15) + if iuv not in iuvs: + iuvs.add(iuv) + payments.append({ + "index": idx + 1, + "iuv": iuv, + "iur": get_random_numeric_string(11), + "pay": total_amount / number_of_payments, + "idTransfer": 1, + "payStatus": "EXECUTED", + "payDate": f"{date}T12:00:00.000Z" + }) + else: + idx -= 1 + return payments + +def get_random_numeric_string(size, dataset = "0123456789"): + random_string = ''.join(random.choices(dataset, k=size)) + return random_string \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6408ae0c..19683857 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 it.gov.pagopa pagopa-fdr - 1.0.23 + 1.0.21-15-PAGOPA-2282 3.11.0 1.18.26 @@ -131,6 +131,10 @@ 1.0.0 provided + + io.quarkus + quarkus-smallrye-fault-tolerance + org.testcontainers junit-jupiter diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java index f9fdbecf..16e581d4 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentInsertEntity.java @@ -8,11 +8,13 @@ import io.quarkus.panache.common.Sort; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import lombok.Data; import lombok.EqualsAndHashCode; import org.bson.codecs.pojo.annotations.BsonProperty; import org.bson.types.ObjectId; +import org.eclipse.microprofile.faulttolerance.Retry; @Data @EqualsAndHashCode(callSuper = true) @@ -67,6 +69,8 @@ public static PanacheQuery findByFdrAndPspId(String fdr, Parameters.with("fdr", fdr).and("pspId", pspId).map()); } + // https://quarkus.io/guides/smallrye-fault-tolerance + @Retry(delay = 500, delayUnit = ChronoUnit.MILLIS) public static long deleteByFdrAndIndexes(String fdr, List indexList) { return delete( "ref_fdr = :fdr and index in :indexes", diff --git a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java index b1493f1d..1c3558b5 100644 --- a/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java +++ b/src/main/java/it/gov/pagopa/fdr/repository/fdr/FdrPaymentPublishEntity.java @@ -6,6 +6,7 @@ import io.quarkus.mongodb.panache.common.MongoEntity; import io.quarkus.panache.common.Parameters; import io.quarkus.panache.common.Sort; +import io.smallrye.mutiny.Uni; import it.gov.pagopa.fdr.repository.fdr.model.PaymentStatusEnumEntity; import java.time.Instant; import java.util.List; @@ -99,22 +100,7 @@ public static PanacheQuery findByPspAndIuvIur( return find(query, sort, params); } - // public static PanacheQuery findByFdrAndPspId( - // String fdr, String pspId, Sort sort) { - // return find( - // "ref_fdr = :fdr and ref_fdr_sender_psp_id = :pspId", - // sort, - // Parameters.with("fdr", fdr).and("pspId", pspId).map()); - // } - // - // public static long deleteByFdrAndPspId(String fdr, String pspId) { - // return delete( - // "ref_fdr = :fdr and ref_fdr_sender_psp_id = :pspId", - // Parameters.with("fdr", fdr).and("pspId", pspId).map()); - // } - // - public static void persistFdrPaymentPublishEntities( - List fdrPaymentPublishEntities) { + public static void persistFdrPaymentPublishEntities(List fdrPaymentPublishEntities) { persist(fdrPaymentPublishEntities); } } diff --git a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java index bc9863f9..044cd462 100644 --- a/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java +++ b/src/main/java/it/gov/pagopa/fdr/service/psps/PspsService.java @@ -10,14 +10,12 @@ import io.quarkus.panache.common.Sort; import it.gov.pagopa.fdr.exception.AppErrorCodeMessageEnum; import it.gov.pagopa.fdr.exception.AppException; -import it.gov.pagopa.fdr.repository.fdr.FdrInsertEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPaymentInsertEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPaymentPublishEntity; -import it.gov.pagopa.fdr.repository.fdr.FdrPublishEntity; +import it.gov.pagopa.fdr.repository.fdr.*; import it.gov.pagopa.fdr.repository.fdr.model.FdrStatusEnumEntity; import it.gov.pagopa.fdr.repository.fdr.projection.FdrInsertProjection; import it.gov.pagopa.fdr.repository.fdr.projection.FdrPublishByPspProjection; import it.gov.pagopa.fdr.repository.fdr.projection.FdrPublishRevisionProjection; +import it.gov.pagopa.fdr.rest.validation.CommonValidationService; import it.gov.pagopa.fdr.service.conversion.ConversionService; import it.gov.pagopa.fdr.service.conversion.message.FdrMessage; import it.gov.pagopa.fdr.service.dto.*; @@ -28,6 +26,7 @@ import it.gov.pagopa.fdr.service.re.model.*; import it.gov.pagopa.fdr.util.AppDBUtil; import it.gov.pagopa.fdr.util.AppMessageUtil; +import it.gov.pagopa.fdr.util.StringUtil; import jakarta.enterprise.context.ApplicationScoped; import java.math.BigDecimal; import java.time.Instant; @@ -35,7 +34,10 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.bson.types.ObjectId; import org.jboss.logging.Logger; import org.jboss.logging.MDC; @@ -273,7 +275,6 @@ public void deletePayment( .build()); } - @WithSpan(kind = SERVER) public void publishByFdr(String action, String pspId, String fdr, boolean internalPublish) { log.infof(AppMessageUtil.logExecute(action)); @@ -304,9 +305,7 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrEntity.getComputedSumPayments()); } - log.debug("FdrInsertEntity PUBLISHED"); - - log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdr, pspId); + log.debugf("Existence check FdrPaymentInsertEntity by fdr[%s], pspId[%s]", StringUtil.sanitize(fdr), StringUtil.sanitize(pspId)); List paymentInsertEntities = FdrPaymentInsertEntity.findByFdrAndPspId(fdr, pspId) .project(FdrPaymentInsertEntity.class) @@ -317,31 +316,71 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern fdrPublishEntity.setUpdated(now); fdrPublishEntity.setPublished(now); fdrPublishEntity.setStatus(FdrStatusEnumEntity.PUBLISHED); - List fdrPaymentPublishEntities = - mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); + List fdrPaymentPublishEntities = mapper.toFdrPaymentPublishEntityList(paymentInsertEntities); - log.info("Starting persistent storage on Mongo of FDR payment entities"); - FdrPaymentPublishEntity.persistFdrPaymentPublishEntities(fdrPaymentPublishEntities); - log.info("End of persistent storage on Mongo of FDR payment entities"); + // writes in fdr_payment_publish collection + this.parallelPersist(fdr, fdrPaymentPublishEntities); - // salva su storage dello storico + // save as JSON file in history storage HistoryBlobBody body = historyService.saveJsonFile(fdrPublishEntity, fdrPaymentPublishEntities); + fdrPublishEntity.setRefJson(body); fdrPublishEntity.persistEntity(); - historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + CompletableFuture executeSaveOnStorage = + CompletableFuture.supplyAsync( + () -> { + historyService.saveOnStorage(fdrPublishEntity, fdrPaymentPublishEntities); + return true; + }); + executeSaveOnStorage + .thenAccept( + value -> { + // queue + this.addToConversionQueue(internalPublish, fdrEntity); + // delete + this.batchDelete(fdr, paymentInsertEntities); + fdrEntity.delete(); + log.infof("Deleted FdrPaymentInsertEntity by fdr[%s], pspId[%s]", StringUtil.sanitize(fdr), StringUtil.sanitize(pspId)); + // re + this.rePublish(fdr, pspId, fdrPublishEntity); + }) + .exceptionally( + e -> { + log.error("Exception during async saveOnStorage: ", e); + return null; + }); + } - log.debug("Delete FdrInsertEntity"); - fdrEntity.delete(); - log.debugf( - "Delete FdrPaymentInsertEntity by fdr[%s], pspId[%s]", fdrEntity.getRevision(), fdr, pspId); - FdrPaymentInsertEntity.deleteByFdrAndPspId(fdr, pspId); + private void batchDelete(String fdr, List paymentInsertEntities) { + int batchSize = 1000; + List> batches = IntStream + .range(0, (paymentInsertEntities.size() + batchSize - 1) / batchSize) + .mapToObj(i -> paymentInsertEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, paymentInsertEntities.size()))) + .toList(); + // sequential stream + batches.forEach(batch -> { + List indexes = paymentInsertEntities.stream().map(FdrPaymentInsertEntity::getIndex).toList(); + FdrPaymentInsertEntity.deleteByFdrAndIndexes(fdr, indexes); + }); + } + private void parallelPersist(String fdr, List fdrPaymentPublishEntities) { + int batchSize = 1000; + List> batchesPublish = IntStream + .range(0, (fdrPaymentPublishEntities.size() + batchSize - 1) / batchSize) + .mapToObj(i -> fdrPaymentPublishEntities.subList(i * batchSize, Math.min((i + 1) * batchSize, fdrPaymentPublishEntities.size()))) + .toList(); + batchesPublish.parallelStream().forEach(FdrPaymentPublishEntity::persistFdrPaymentPublishEntities); + log.debugf("Published fdrPaymentPublishEntities of fdr[%s]", StringUtil.sanitize(fdr)); + } + + private void addToConversionQueue(boolean internalPublish, FdrInsertEntity fdrEntity) { // add to conversion queue if (internalPublish) { - log.debug("NOT Add FdrInsertEntity in queue fdr message"); + log.debugf("NOT Add FdrInsertEntity in queue fdr message"); } else { - log.debug("Add FdrInsertEntity in queue fdr message"); + log.debugf("Starting add FdrInsertEntity in queue fdr message"); conversionQueue.addQueueFlowMessage( FdrMessage.builder() .fdr(fdrEntity.getFdr()) @@ -350,8 +389,11 @@ public void publishByFdr(String action, String pspId, String fdr, boolean intern .retry(0L) .revision(fdrEntity.getRevision()) .build()); + log.debugf("End add FdrInsertEntity in queue fdr message"); } + } + private void rePublish(String fdr, String pspId, FdrPublishEntity fdrPublishEntity) { String sessionId = (String) MDC.get(TRX_ID); MDC.put(EVENT_CATEGORY, EventTypeEnum.INTERNAL.name()); reService.sendEvent( diff --git a/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java b/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java index 519e64c0..b624bfb9 100644 --- a/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java +++ b/src/main/java/it/gov/pagopa/fdr/util/StringUtil.java @@ -17,4 +17,11 @@ public static byte[] zip(String str) throws IOException { bais.close(); return compressed; } + + // Replace newline, carriage return, tab, single quote, double quote, and backslash characters + public static String sanitize(String input) { + if (input == null) + return null; + return input.replaceAll("[\\n\\r\\t'\"\\\\]", "_"); + } }