faster docker build, added ebicsResponse trace

.github/workflows/dockerhub.yml

@@ -7,6 +7,9 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
+    - name: Bootstrap - neeed by LibeuFin to inititalize the sub-repositories
+      run: ./bootstrap
+
     - name: Log in to Docker Hub
       uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
       with:
@@ -27,5 +30,3 @@ jobs:
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
-
-

.gitignore

@@ -20,3 +20,7 @@ __pycache__
 *.mk
 util/src/main/resources/version.txt
 .gitconfig
+*/bin
+.gitconfig
+*/spx/*
+spx

Dockerfile

@@ -1,21 +1,45 @@
 FROM zenika/kotlin:1.4.20-M2-jdk11-slim as build
+#
+#  IMPORTANT - you need to call ./bootstrap in order to inititalize the sub-repo of Libeufin
+#  Ohterwise the build will fail with
+#  failed to solve: failed to compute cache key: failed to calculate checksum  ..."/build-system/taler-build-scripts/configure": not found
+#
 RUN apt update
+# needed for nexus/sandbox
 RUN apt install -y python3 python3-pip git
 RUN pip3 install click requests
-RUN mkdir /app
-COPY ./ /app/
-WORKDIR /app/
-RUN chmod a+x bootstrap; ./bootstrap
-RUN chmod a+x configure; ./configure
-RUN make install
-# needed for stating the UI to use the sandbox - LibFinEu/frontend
+# needed for starting the UI to use the sandbox - LibFinEu/frontend
 RUN apt-get install -y wget curl nodejs yarnpkg npm 
 RUN apt-get install postgresql-client -y
 #  install versions according to LibFinEu/frontend/README.md
 RUN npm install -g n
 RUN n 10.16.0
 RUN npm install -g [email protected]
 RUN npm install --global [email protected]
+RUN yarnpkg global add [email protected]
+# moved ahead so that debugging kotlin is faster
+RUN mkdir /app; ls -la /app/
+COPY gradlew build.gradle gradle.properties Makefile settings.gradle  /app/
+COPY ./presentation /app/presentation
+COPY ./frontend /app/frontend
+WORKDIR /app/
 RUN yarn --cwd /app/frontend/ install
-# RUN yarnpkg --cwd /app/frontend/ build
-RUN yarnpkg global add serve
+
+# setup system
+ARG CACHEBUST=10
+COPY ./build-system /app/build-system
+COPY ./cli /app/cli
+COPY ./contrib /app/contrib
+COPY ./debian /app/debian
+COPY ./nexus /app/nexus
+COPY ./parsing-tests /app/parsing-tests
+COPY ./sandbox /app/sandbox
+COPY ./util /app/util
+COPY ./gradle /app/gradle
+COPY build-system/taler-build-scripts/configure /app/configure
+
+RUN chmod a+x configure; ./configure
+RUN make all
+RUN make install
+
+# RUN yarnpkg --cwd /app/frontend/ build

nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsClient.kt

@@ -23,31 +23,48 @@
 package tech.libeufin.nexus.ebics
 
 import io.ktor.client.HttpClient
-import io.ktor.client.request.post
-import io.ktor.http.HttpStatusCode
+import io.ktor.client.features.*
+import io.ktor.client.request.*
+import io.ktor.http.*
+import io.ktor.util.*
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import tech.libeufin.nexus.NexusError
 import tech.libeufin.util.*
 import java.util.*
+import java.io.File
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import tech.libeufin.nexus.server.setTransactionId
 
 private val logger: Logger = LoggerFactory.getLogger("tech.libeufin.util")
 
 private suspend inline fun HttpClient.postToBank(url: String, body: String): String {
     logger.debug("Posting: $body")
     if (!XMLUtil.validateFromString(body)) throw NexusError(
-        HttpStatusCode.InternalServerError, "EBICS (outgoing) document is invalid"
+        HttpStatusCode.InternalServerError,
+        "EBICS (outgoing) document is invalid"
     )
     val response: String = try {
-        this.post<String>(
+        this.post(
             urlString = url,
             block = {
                 this.body = body
             }
         )
-    } catch (e: Exception) {
-        logger.warn("Exception during request", e)
-        throw NexusError(HttpStatusCode.InternalServerError, "Cannot reach the bank")
+    } catch (e: ClientRequestException) {
+        logger.error(e.message)
+        throw NexusError(
+            HttpStatusCode.BadGateway,
+            e.message
+        )
+    }
+    catch (e: Exception) {
+        logger.error("Exception during request", e)
+        throw NexusError(
+            HttpStatusCode.BadGateway,
+            e.message ?: "Could not reach the bank"
+        )
     }
     logger.debug("Receiving: $response")
     return response
@@ -56,16 +73,44 @@ private suspend inline fun HttpClient.postToBank(url: String, body: String): Str
 sealed class EbicsDownloadResult
 
 class EbicsDownloadSuccessResult(
-    val orderData: ByteArray
+    val orderData: ByteArray,
+    val transactionId: String
 ) : EbicsDownloadResult()
 
 /**
- * Some bank-technical error occured.
+ * Some bank-technical error occured.w
  */
 class EbicsDownloadBankErrorResult(
     val returnCode: EbicsReturnCode
 ) : EbicsDownloadResult()
 
+
+fun writeResponseToFile( transactionId: String, initResponseStr: String) {
+    // Get base directory from environment variable TRACE_DIR
+    setTransactionId(transactionId)
+    val baseDirectory = System.getenv("TRACE_DIR") ?: System.getProperty("user.dir")
+
+    // Check if directory "trace" exists, create it if not
+    val traceDirectory = File("$baseDirectory/trace")
+    if (!traceDirectory.exists()) {
+        traceDirectory.mkdirs()
+    }
+
+    // Get current date and time
+    val currentDateTime = LocalDateTime.now()
+    val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")
+    val formattedDateTime = currentDateTime.format(formatter)
+
+    // Construct file name with transactionId and current date/time
+    val fileName = "${traceDirectory.absolutePath}/${transactionId}_$formattedDateTime.xml"
+
+    // Write response to file
+    val file = File(fileName)
+    file.writeText(initResponseStr)
+    println("Response written to file: $fileName")
+}
+
+
 /**
  * Do an EBICS download transaction.  This includes the initialization phase, transaction phase
  * and receipt phase.
@@ -90,7 +135,8 @@ suspend fun doEbicsDownloadTransaction(
         else -> {
             throw EbicsProtocolError(
                 HttpStatusCode.InternalServerError,
-                "unexpected return code ${initResponse.technicalReturnCode}"
+                "unexpected return code ${initResponse.technicalReturnCode}",
+                initResponse.technicalReturnCode
             )
         }
     }
@@ -110,7 +156,9 @@ suspend fun doEbicsDownloadTransaction(
             HttpStatusCode.InternalServerError,
             "initial response must contain transaction ID"
         )
-
+    // Write response to file wasa
+    writeResponseToFile(transactionID,initResponseStr)
+
     val encryptionInfo = initResponse.dataEncryptionInfo
         ?: throw NexusError(HttpStatusCode.InternalServerError, "initial response did not contain encryption info")
 
@@ -177,7 +225,7 @@ suspend fun doEbicsDownloadTransaction(
             throw NexusError(HttpStatusCode.InternalServerError, "unexpected return code")
         }
     }
-    return EbicsDownloadSuccessResult(respPayload)
+    return EbicsDownloadSuccessResult(respPayload, transactionID)
 }
 
 
@@ -237,8 +285,7 @@ suspend fun doEbicsUploadTransaction(
 suspend fun doEbicsHostVersionQuery(client: HttpClient, ebicsBaseUrl: String, ebicsHostId: String): EbicsHevDetails {
     val ebicsHevRequest = makeEbicsHEVRequestRaw(ebicsHostId)
     val resp = client.postToBank(ebicsBaseUrl, ebicsHevRequest)
-    val versionDetails = parseEbicsHEVResponse(resp)
-    return versionDetails
+    return parseEbicsHEVResponse(resp)
 }
 
 suspend fun doEbicsIniRequest(
@@ -250,8 +297,7 @@ suspend fun doEbicsIniRequest(
         subscriberDetails.ebicsUrl,
         request
     )
-    val resp = parseAndDecryptEbicsKeyManagementResponse(subscriberDetails, respStr)
-    return resp
+    return parseAndDecryptEbicsKeyManagementResponse(subscriberDetails, respStr)
 }
 
 suspend fun doEbicsHiaRequest(
@@ -263,8 +309,7 @@ suspend fun doEbicsHiaRequest(
         subscriberDetails.ebicsUrl,
         request
     )
-    val resp = parseAndDecryptEbicsKeyManagementResponse(subscriberDetails, respStr)
-    return resp
+    return parseAndDecryptEbicsKeyManagementResponse(subscriberDetails, respStr)
 }
 
 

nexus/src/main/kotlin/tech/libeufin/nexus/ebics/EbicsNexus.kt

@@ -70,7 +70,6 @@ private data class EbicsFetchSpec(
     val orderParams: EbicsOrderParams
 )
 
-// Moved eventually in a tucked "camt" file.
 fun storeCamt(bankConnectionId: String, camt: String, historyType: String) {
     val camt53doc = XMLUtil.parseStringIntoDom(camt)
     val msgId = camt53doc.pickStringWithRootNs("/*[1]/*[1]/root:GrpHdr/root:MsgId")
@@ -104,12 +103,24 @@ private suspend fun fetchEbicsC5x(
     subscriberDetails: EbicsClientSubscriberDetails
 ) {
     logger.debug("Requesting $historyType")
-    val response = doEbicsDownloadTransaction(
-        client,
-        subscriberDetails,
-        historyType,
-        orderParams
-    )
+    val response = try {
+        doEbicsDownloadTransaction(
+            client,
+            subscriberDetails,
+            historyType,
+            orderParams
+        )
+    } catch (e: EbicsProtocolError) {
+        /**
+         * This error type is not an actual error in this handler.
+         */
+        if (e.ebicsTechnicalCode == EbicsReturnCode.EBICS_NO_DOWNLOAD_DATA_AVAILABLE) {
+            logger.info("Could not find new transactions to download")
+            return
+        }
+        // re-throw in any other error case.
+        throw e
+    }
     when (historyType) {
         "C52" -> {
         }
@@ -122,7 +133,7 @@ private suspend fun fetchEbicsC5x(
     when (response) {
         is EbicsDownloadSuccessResult -> {
             response.orderData.unzipWithLambda {
-                logger.debug("Camt entry: ${it.second}")
+                logger.debug("Camt entry (filename (in the Zip archive): ${it.first}): ${it.second}")
                 storeCamt(bankConnectionId, it.second, historyType)
             }
         }
@@ -302,7 +313,7 @@ fun Route.ebicsBankConnectionRoutes(client: HttpClient) {
     }
 
     post("/download/{msgtype}") {
-        val orderType = requireNotNull(call.parameters["msgtype"]).toUpperCase(Locale.ROOT)
+        val orderType = requireNotNull(call.parameters["msgtype"]).uppercase(Locale.ROOT)
         if (orderType.length != 3) {
             throw NexusError(HttpStatusCode.BadRequest, "ebics order type must be three characters")
         }
@@ -596,7 +607,7 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
             it.add(Paragraph("Verschlüsselungsschlüssel").setFontSize(24f))
             writeCommon(it)
             it.add(Paragraph("Öffentlicher Schlüssel (Public encryption key)"))
-            writeKey(it, ebicsSubscriber.customerSignPriv)
+            writeKey(it, ebicsSubscriber.customerEncPriv)
             it.add(Paragraph("\n"))
             writeSigLine(it)
         }
@@ -795,6 +806,13 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
         }
         if (subscriber.ebicsIniState == EbicsInitState.UNKNOWN || subscriber.ebicsHiaState == EbicsInitState.UNKNOWN) {
             if (tentativeHpb(client, connId)) {
+                /**
+                 * NOTE/FIXME: in case the HIA/INI did succeed (state is UNKNOWN but Sandbox
+                 * has somehow the keys), here the state should be set to SENT, because later -
+                 * when the Sandbox will respond to the INI/HIA requests - we'll get a
+                 * EBICS_INVALID_USER_OR_USER_STATE.  Hence, the state will never switch to
+                 * SENT again.
+                 */
                 return
             }
         }
@@ -815,7 +833,7 @@ class EbicsBankConnectionProtocol: BankConnectionProtocol {
         val hpbData = try {
             doEbicsHpbRequest(client, subscriber)
         } catch (e: EbicsProtocolError) {
-            logger.warn("failed hpb request", e)
+            logger.warn("failed HPB request", e)
             null
         }
         transaction {