diff --git a/build.sbt b/build.sbt index e0ad0a586d5..32f2560e0e1 100644 --- a/build.sbt +++ b/build.sbt @@ -644,21 +644,16 @@ lazy val flinkPeriodicDeploymentManager = (project in flink("management/periodic name := "nussknacker-flink-periodic-manager", libraryDependencies ++= { Seq( - "org.typelevel" %% "cats-core" % catsV % Provided, - "com.typesafe.slick" %% "slick" % slickV % Provided, - "com.typesafe.slick" %% "slick-hikaricp" % slickV % "provided, test", - "com.github.tminglei" %% "slick-pg" % slickPgV, - "org.hsqldb" % "hsqldb" % hsqldbV % Test, - "org.flywaydb" % "flyway-core" % flywayV % Provided, - "com.cronutils" % "cron-utils" % cronParserV, - "com.typesafe.akka" %% "akka-actor" % akkaV, - "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, - "com.dimafeng" %% "testcontainers-scala-scalatest" % testContainersScalaV % Test, - "com.dimafeng" %% "testcontainers-scala-postgresql" % testContainersScalaV % Test, + "com.typesafe.slick" %% "slick-hikaricp" % slickV % "provided, test", + "org.hsqldb" % "hsqldb" % hsqldbV % Test, + "com.typesafe.akka" %% "akka-testkit" % akkaV % Test, + "com.dimafeng" %% "testcontainers-scala-scalatest" % testContainersScalaV % Test, + "com.dimafeng" %% "testcontainers-scala-postgresql" % testContainersScalaV % Test, ) } ) .dependsOn( + commonPeriodicDeploymentManager, flinkDeploymentManager, deploymentManagerApi % Provided, scenarioCompiler % Provided, @@ -1790,6 +1785,25 @@ lazy val commonComponentsTests = (project in engine("common/components-tests")) flinkComponentsTestkit % Test ) +lazy val commonPeriodicDeploymentManager = (project in engine("common/periodic-deployment-manager")) + .settings(commonSettings) + .settings(publishAssemblySettings: _*) + .settings( + name := "nussknacker-common-periodic-deployment-manager", + libraryDependencies ++= { + Seq( + "com.typesafe.akka" %% "akka-actor" % akkaV, + "org.typelevel" %% "cats-core" % catsV % Provided, + "com.cronutils" % "cron-utils" % cronParserV, + "com.softwaremill.retry" %% "retry" % retryV, + "com.github.tminglei" %% "slick-pg" % slickPgV, + ) + } + ) + .dependsOn( + deploymentManagerApi % Provided, + ) + lazy val flinkBaseComponents = (project in flink("components/base")) .settings(commonSettings) .settings(assemblyNoScala("flinkBase.jar"): _*) @@ -2069,7 +2083,7 @@ lazy val designer = (project in file("designer/server")) liteEmbeddedDeploymentManager % Provided, liteK8sDeploymentManager % Provided, developmentTestsDeploymentManager % Provided, - flinkPeriodicDeploymentManager % Provided, + flinkPeriodicDeploymentManager % "provided, test->test", schemedKafkaComponentsUtils % Provided, ) diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala index bf8fed6e669..4f7242b16be 100644 --- a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/DeploymentManagerDependencies.scala @@ -1,12 +1,13 @@ package pl.touk.nussknacker.engine import akka.actor.ActorSystem +import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId} +import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} import pl.touk.nussknacker.engine.api.deployment.{ ProcessingTypeActionService, ProcessingTypeDeployedScenariosProvider, ScenarioActivityManager } -import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId} import sttp.client3.SttpBackend import scala.concurrent.{ExecutionContext, Future} @@ -15,6 +16,7 @@ case class DeploymentManagerDependencies( deployedScenariosProvider: ProcessingTypeDeployedScenariosProvider, actionService: ProcessingTypeActionService, scenarioActivityManager: ScenarioActivityManager, + periodicProcessesManagerProvider: PeriodicProcessesManagerProvider, executionContext: ExecutionContext, actorSystem: ActorSystem, sttpBackend: SttpBackend[Future, Any], diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala new file mode 100644 index 00000000000..457072ae5a0 --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManager.scala @@ -0,0 +1,150 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic + +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess + +import java.time.LocalDateTime +import scala.concurrent.Future + +trait PeriodicProcessesManager { + + def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] + + def markInactive(processId: PeriodicProcessId): Future[Unit] + + def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int, + ): Future[PeriodicProcessDeployment] + + def findProcessData(id: PeriodicProcessDeploymentId): Future[PeriodicProcessDeployment] + + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] + + def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] + + def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] + + def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] + + def getSchedulesState( + scenarioName: ProcessName + ): Future[SchedulesState] + + def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] + + def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] + + def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] + + def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId, + ): Future[Option[CanonicalProcess]] + +} + +object PeriodicProcessesManager { + + sealed trait ScheduleProperty + + sealed trait SingleScheduleProperty extends ScheduleProperty + + case class MultipleScheduleProperty(schedules: Map[String, SingleScheduleProperty]) extends ScheduleProperty + + case class CronScheduleProperty(labelOrCronExpr: String) extends SingleScheduleProperty + +} + +object NoOpPeriodicProcessesManager extends PeriodicProcessesManager { + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] = notImplemented + + override def markInactive(processId: PeriodicProcessId): Future[Unit] = notImplemented + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Future[PeriodicProcessDeployment] = notImplemented + + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = notImplemented + + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = notImplemented + + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = notImplemented + + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = notImplemented + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] = notImplemented + + override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = notImplemented + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = notImplemented + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = notImplemented + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = notImplemented + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = notImplemented + + private def notImplemented: Future[Nothing] = + Future.failed(new NotImplementedError()) + +} diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala new file mode 100644 index 00000000000..c756a754d7f --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/PeriodicProcessesManagerProvider.scala @@ -0,0 +1,19 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic + +trait PeriodicProcessesManagerProvider { + + def provide( + deploymentManagerName: String, + processingType: String, + ): PeriodicProcessesManager + +} + +object NoOpPeriodicProcessesManagerProvider extends PeriodicProcessesManagerProvider { + + override def provide( + deploymentManagerName: String, + processingType: String + ): PeriodicProcessesManager = NoOpPeriodicProcessesManager + +} diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala new file mode 100644 index 00000000000..84811ae0c9a --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/DeploymentWithRuntimeParams.scala @@ -0,0 +1,11 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic.model + +import pl.touk.nussknacker.engine.api.ProcessVersion + +case class DeploymentWithRuntimeParams( + processVersion: ProcessVersion, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, +) + +final case class RuntimeParams(params: Map[String, String]) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala similarity index 55% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala rename to designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala index effee642b22..ab1c666a008 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcess.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcess.scala @@ -1,17 +1,16 @@ -package pl.touk.nussknacker.engine.management.periodic.model +package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.management.periodic.ScheduleProperty -import slick.lifted.MappedTo +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.ScheduleProperty import java.time.LocalDateTime -case class PeriodicProcessId(value: Long) extends MappedTo[Long] +case class PeriodicProcessId(value: Long) -case class PeriodicProcess[ProcessRep]( +case class PeriodicProcess( id: PeriodicProcessId, - deploymentData: DeploymentWithJarData[ProcessRep], + deploymentData: DeploymentWithRuntimeParams, scheduleProperty: ScheduleProperty, active: Boolean, createdAt: LocalDateTime, diff --git a/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala new file mode 100644 index 00000000000..e67b46142dc --- /dev/null +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/PeriodicProcessDeployment.scala @@ -0,0 +1,42 @@ +package pl.touk.nussknacker.engine.api.deployment.periodic.model + +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus + +import java.time.LocalDateTime + +// TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData +case class PeriodicProcessDeployment( + id: PeriodicProcessDeploymentId, + periodicProcess: PeriodicProcess, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: ScheduleName, + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + state: PeriodicProcessDeploymentState +) { + + def display: String = + s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" + +} + +case class PeriodicProcessDeploymentState( + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + status: PeriodicProcessDeploymentStatus +) + +case class PeriodicProcessDeploymentId(value: Long) { + override def toString: String = value.toString +} + +object PeriodicProcessDeploymentStatus extends Enumeration { + type PeriodicProcessDeploymentStatus = Value + + val Scheduled, Deployed, Finished, Failed, RetryingDeploy, FailedOnDeploy = Value +} + +case class ScheduleName(value: Option[String]) { + def display: String = value.getOrElse("[default]") +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala similarity index 71% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala rename to designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala index ff8aeaac666..ecfddb07ab0 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/SchedulesState.scala +++ b/designer/deployment-manager-api/src/main/scala/pl/touk/nussknacker/engine/api/deployment/periodic/model/SchedulesState.scala @@ -1,7 +1,6 @@ -package pl.touk.nussknacker.engine.management.periodic.model +package pl.touk.nussknacker.engine.api.deployment.periodic.model import pl.touk.nussknacker.engine.api.process.ProcessName -import pl.touk.nussknacker.engine.management.periodic.db.{PeriodicProcessDeploymentEntity, PeriodicProcessesRepository} import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import java.time.LocalDateTime @@ -35,7 +34,7 @@ case class SchedulesState(schedules: Map[ScheduleId, ScheduleData]) { // For most operations it will contain only one latest deployment but for purpose of statuses of historical deployments // it has list instead of one element. // This structure should contain SingleScheduleProperty as well. See note above -case class ScheduleData(process: PeriodicProcess[Unit], latestDeployments: List[ScheduleDeploymentData]) +case class ScheduleData(process: PeriodicProcess, latestDeployments: List[ScheduleDeploymentData]) // To identify schedule we need scheduleName - None for SingleScheduleProperty and Some(key) for MultipleScheduleProperty keys // Also we need PeriodicProcessId to distinguish between active schedules and some inactive from the past for the same PeriodicProcessId @@ -53,41 +52,17 @@ case class ScheduleDeploymentData( ) { def toFullDeploymentData( - process: PeriodicProcess[Unit], + process: PeriodicProcess, scheduleName: ScheduleName - ): PeriodicProcessDeployment[Unit] = + ): PeriodicProcessDeployment = PeriodicProcessDeployment(id, process, createdAt, runAt, scheduleName, retriesLeft, nextRetryAt, state) def display = s"deploymentId=$id" } -object ScheduleDeploymentData { - - def apply(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { - ScheduleDeploymentData( - deployment.id, - deployment.createdAt, - deployment.runAt, - deployment.deployedAt, - deployment.retriesLeft, - deployment.nextRetryAt, - PeriodicProcessesRepository.createPeriodicDeploymentState(deployment) - ) - } - -} - // These below are temporary structures, see notice next to SchedulesState case class PeriodicProcessScheduleData( - process: PeriodicProcess[Unit], - deployments: List[PeriodicProcessDeployment[Unit]] -) { - def existsDeployment(predicate: PeriodicProcessDeployment[Unit] => Boolean): Boolean = deployments.exists(predicate) - - def display: String = { - val deploymentsForSchedules = deployments.map(_.display) - s"processName=${process.processVersion.processName}, deploymentsForSchedules=$deploymentsForSchedules" - } - -} + process: PeriodicProcess, + deployments: List[PeriodicProcessDeployment] +) diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/custom-actions/batch-instant.svg b/designer/server/src/main/resources/web/static/assets/custom-actions/batch-instant.svg similarity index 100% rename from engine/flink/management/periodic/src/main/resources/web/static/assets/custom-actions/batch-instant.svg rename to designer/server/src/main/resources/web/static/assets/custom-actions/batch-instant.svg diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/states/scheduled.svg b/designer/server/src/main/resources/web/static/assets/states/scheduled.svg similarity index 100% rename from engine/flink/management/periodic/src/main/resources/web/static/assets/states/scheduled.svg rename to designer/server/src/main/resources/web/static/assets/states/scheduled.svg diff --git a/engine/flink/management/periodic/src/main/resources/web/static/assets/states/wait-reschedule.svg b/designer/server/src/main/resources/web/static/assets/states/wait-reschedule.svg similarity index 100% rename from engine/flink/management/periodic/src/main/resources/web/static/assets/states/wait-reschedule.svg rename to designer/server/src/main/resources/web/static/assets/states/wait-reschedule.svg diff --git a/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala new file mode 100644 index 00000000000..fd1b42362f4 --- /dev/null +++ b/designer/server/src/main/scala/db/migration/V1_060__PeriodicDeploymentManagerTablesDefinition.scala @@ -0,0 +1,190 @@ +package db.migration + +import com.typesafe.scalalogging.LazyLogging +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition.Definitions +import pl.touk.nussknacker.ui.db.migration.SlickMigration +import slick.jdbc.JdbcProfile +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime +import java.util.UUID +import scala.concurrent.ExecutionContext.Implicits.global + +trait V1_060__PeriodicDeploymentManagerTablesDefinition extends SlickMigration with LazyLogging { + + import profile.api._ + + private val definitions = new Definitions(profile) + + override def migrateActions: DBIOAction[Any, NoStream, Effect.All] = { + logger.info("Starting migration V1_056__CreateScenarioActivitiesDefinition") + for { + _ <- definitions.periodicProcessesTable.schema.create + _ <- definitions.periodicProcessDeploymentsTable.schema.create + } yield () + } + +} + +object V1_060__PeriodicDeploymentManagerTablesDefinition { + + class Definitions(val profile: JdbcProfile) { + import profile.api._ + + val periodicProcessDeploymentsTable = TableQuery[PeriodicProcessDeploymentsTable] + + class PeriodicProcessDeploymentsTable(tag: Tag) + extends Table[PeriodicProcessDeploymentEntity](tag, "periodic_process_deployments") { + + def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) + + def periodicProcessId: Rep[Long] = column[Long]("periodic_process_id", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def runAt: Rep[LocalDateTime] = column[LocalDateTime]("run_at", NotNull) + + def scheduleName: Rep[Option[String]] = column[Option[String]]("schedule_name") + + def deployedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("deployed_at") + + def completedAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("completed_at") + + def retriesLeft: Rep[Int] = column[Int]("retries_left") + + def nextRetryAt: Rep[Option[LocalDateTime]] = column[Option[LocalDateTime]]("next_retry_at") + + def status: Rep[String] = column[String]("status", NotNull) + + override def * : ProvenShape[PeriodicProcessDeploymentEntity] = ( + id, + periodicProcessId, + createdAt, + runAt, + scheduleName, + deployedAt, + completedAt, + retriesLeft, + nextRetryAt, + status + ) <> + ((PeriodicProcessDeploymentEntity.apply _).tupled, PeriodicProcessDeploymentEntity.unapply) + + } + + case class PeriodicProcessDeploymentEntity( + id: Long, + periodicProcessId: Long, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: Option[String], + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + status: String + ) + + val periodicProcessesTable = TableQuery[PeriodicProcessesTable] + + class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + + def id: Rep[Long] = column[Long]("id", O.PrimaryKey, O.AutoInc) + + def processName: Rep[String] = column[String]("process_name", NotNull) + + def processVersionId: Rep[Long] = column[Long]("process_version_id", NotNull) + + def processingType: Rep[String] = column[String]("processing_type", NotNull) + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + // This is a legacy column left after migrating periodic processes to core + // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column + def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") + + def runtimeParams: Rep[String] = column[String]("runtime_params") + + def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) + + def active: Rep[Boolean] = column[Boolean]("active", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def processActionId: Rep[Option[UUID]] = column[Option[UUID]]("process_action_id") + + override def * : ProvenShape[PeriodicProcessEntity] = ( + id, + processName, + processVersionId, + processingType, + inputConfigDuringExecutionJson, + jarFileName, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + tuple => + PeriodicProcessEntity( + id = tuple._1, + processName = tuple._2, + processVersionId = tuple._3, + processingType = tuple._4, + inputConfigDuringExecutionJson = tuple._5, + runtimeParams = tuple._7, + scheduleProperty = tuple._8, + active = tuple._9, + createdAt = tuple._10, + processActionId = tuple._11, + ), + (e: PeriodicProcessEntity) => + PeriodicProcessEntity.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + None, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + case class PeriodicProcessEntity( + id: Long, + processName: String, + processVersionId: Long, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: String, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[UUID] + ) + + } + +} diff --git a/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala b/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala new file mode 100644 index 00000000000..3e485f8f4da --- /dev/null +++ b/designer/server/src/main/scala/db/migration/hsql/V1_060__PeriodicDeploymentManagerTables.scala @@ -0,0 +1,8 @@ +package db.migration.hsql + +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition +import slick.jdbc.{HsqldbProfile, JdbcProfile} + +class V1_060__PeriodicDeploymentManagerTables extends V1_060__PeriodicDeploymentManagerTablesDefinition { + override protected lazy val profile: JdbcProfile = HsqldbProfile +} diff --git a/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala b/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala new file mode 100644 index 00000000000..b5f19970a7d --- /dev/null +++ b/designer/server/src/main/scala/db/migration/postgres/V1_060__PeriodicDeploymentManagerTables.scala @@ -0,0 +1,8 @@ +package db.migration.postgres + +import db.migration.V1_060__PeriodicDeploymentManagerTablesDefinition +import slick.jdbc.{JdbcProfile, PostgresProfile} + +class V1_060__PeriodicDeploymentManagerTables extends V1_060__PeriodicDeploymentManagerTablesDefinition { + override protected lazy val profile: JdbcProfile = PostgresProfile +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala similarity index 82% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala index f9b1bacb69b..f75cbff1395 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessDeploymentsTable.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessDeploymentsTable.scala @@ -1,11 +1,7 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.ui.db.entity -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentStatus, - PeriodicProcessId -} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessDeploymentId, PeriodicProcessDeploymentStatus, PeriodicProcessId} import slick.jdbc.{JdbcProfile, JdbcType} import slick.lifted.ProvenShape import slick.sql.SqlProfile.ColumnOption.NotNull @@ -14,10 +10,11 @@ import java.time.LocalDateTime trait PeriodicProcessDeploymentsTableFactory extends PeriodicProcessesTableFactory { - protected val profile: JdbcProfile - import profile.api._ + implicit val periodicProcessDeploymentIdMapping: BaseColumnType[PeriodicProcessDeploymentId] = + MappedColumnType.base[PeriodicProcessDeploymentId, Long](_.value, PeriodicProcessDeploymentId.apply) + implicit val periodicProcessDeploymentStatusColumnTyped: JdbcType[PeriodicProcessDeploymentStatus] = MappedColumnType.base[PeriodicProcessDeploymentStatus, String](_.toString, PeriodicProcessDeploymentStatus.withName) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala new file mode 100644 index 00000000000..0d3b85155c4 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/db/entity/PeriodicProcessesTable.scala @@ -0,0 +1,140 @@ +package pl.touk.nussknacker.ui.db.entity + +import io.circe.Decoder +import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{PeriodicProcessId, RuntimeParams} +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} +import slick.jdbc.JdbcProfile +import slick.lifted.ProvenShape +import slick.sql.SqlProfile.ColumnOption.NotNull + +import java.time.LocalDateTime +import java.util.UUID + +trait PeriodicProcessesTableFactory extends BaseEntityFactory{ + + protected val profile: JdbcProfile + + import profile.api._ + + implicit val periodicProcessIdMapping: BaseColumnType[PeriodicProcessId] = + MappedColumnType.base[PeriodicProcessId, Long](_.value, PeriodicProcessId.apply) + + private implicit val ProcessActionIdTypedType: BaseColumnType[ProcessActionId] = + MappedColumnType.base[ProcessActionId, UUID]( + _.value, + ProcessActionId(_) + ) + + implicit val runtimeParamsTypedType: BaseColumnType[RuntimeParams] = + MappedColumnType.base[RuntimeParams, String]( + _.params.asJson.noSpaces, + jsonStr => + io.circe.parser.parse(jsonStr).flatMap(Decoder[Map[String, String]].decodeJson) match { + case Right(params) => RuntimeParams(params) + case Left(error) => throw error + } + ) + + class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { + + def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) + + def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) + + def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) + + def processingType: Rep[String] = column[String]("processing_type", NotNull) + + def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) + + // This is a legacy column left after migrating periodic processes to core + // The periodic deployment manager is now decoupled from Flink, and its runtime params are stored in runtime_params column + def jarFileName: Rep[Option[String]] = column[Option[String]]("jar_file_name") + + def runtimeParams: Rep[RuntimeParams] = column[RuntimeParams]("runtime_params") + + def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) + + def active: Rep[Boolean] = column[Boolean]("active", NotNull) + + def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) + + def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") + + override def * : ProvenShape[PeriodicProcessEntity] = ( + id, + processName, + processVersionId, + processingType, + inputConfigDuringExecutionJson, + jarFileName, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) <> ( + tuple => + PeriodicProcessEntity( + id = tuple._1, + processName = tuple._2, + processVersionId = tuple._3, + processingType = tuple._4, + inputConfigDuringExecutionJson = tuple._5, + runtimeParams = + RuntimeParams(tuple._6.map(f => Map("jarFileName" -> f)).getOrElse(Map.empty) ++ tuple._7.params), + scheduleProperty = tuple._8, + active = tuple._9, + createdAt = tuple._10, + processActionId = tuple._11, + ), + (e: PeriodicProcessEntity) => + PeriodicProcessEntity.unapply(e).map { + case ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) => + ( + id, + processName, + versionId, + processingType, + inputConfigDuringExecutionJson, + None, + runtimeParams, + scheduleProperty, + active, + createdAt, + processActionId + ) + } + ) + + } + + object PeriodicProcesses extends TableQuery(new PeriodicProcessesTable(_)) + +} + +final case class PeriodicProcessEntity( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] +) diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala new file mode 100644 index 00000000000..957b11dc6ba --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManager.scala @@ -0,0 +1,106 @@ +package pl.touk.nussknacker.ui.process.periodic + +import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository + +import java.time.LocalDateTime +import scala.concurrent.Future + +class RepositoryBasedPeriodicProcessesManager( + deploymentManagerName: String, + processingType: String, + periodicProcessesRepository: PeriodicProcessesRepository, +) extends PeriodicProcessesManager { + + import periodicProcessesRepository._ + + override def create( + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + ): Future[PeriodicProcess] = + periodicProcessesRepository + .create(deploymentWithRuntimeParams, scheduleProperty, processActionId, processingType) + .run + + override def markInactive(processId: PeriodicProcessId): Future[Unit] = + periodicProcessesRepository.markInactive(processId).run + + override def schedule( + id: PeriodicProcessId, + scheduleName: ScheduleName, + runAt: LocalDateTime, + deployMaxRetries: Int + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.schedule(id, scheduleName, runAt, deployMaxRetries).run + + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = + periodicProcessesRepository.findProcessData(id).run + + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeDeployed(processingType).run + + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = + periodicProcessesRepository.findToBeRetried(processingType).run + + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markDeployed(id).run + + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFinished(id).run + + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = + periodicProcessesRepository.markFailed(id).run + + override def markFailedOnDeployWithStatus( + id: PeriodicProcessDeploymentId, + status: PeriodicProcessDeploymentStatus, + deployRetries: Int, + retryAt: Option[LocalDateTime] + ): Future[Unit] = periodicProcessesRepository.markFailedOnDeployWithStatus(id, status, deployRetries, retryAt).run + + override def getSchedulesState(scenarioName: ProcessName): Future[SchedulesState] = + periodicProcessesRepository.getSchedulesState(scenarioName).run + + override def getLatestDeploymentsForActiveSchedules( + processName: ProcessName, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount, processingType) + .run + + override def getLatestDeploymentsForLatestInactiveSchedules( + processName: ProcessName, + inactiveProcessesMaxCount: Int, + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = + periodicProcessesRepository + .getLatestDeploymentsForLatestInactiveSchedules( + processName, + inactiveProcessesMaxCount, + deploymentsPerScheduleMaxCount, + processingType, + ) + .run + + override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = periodicProcessesRepository + .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus(expectedDeploymentStatuses, processingType) + .run + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = + periodicProcessesRepository.fetchCanonicalProcess(processName, versionId).run + +} diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala new file mode 100644 index 00000000000..f2b576e35a0 --- /dev/null +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/periodic/RepositoryBasedPeriodicProcessesManagerProvider.scala @@ -0,0 +1,21 @@ +package pl.touk.nussknacker.ui.process.periodic + +import pl.touk.nussknacker.engine.api.deployment.periodic.{PeriodicProcessesManager, PeriodicProcessesManagerProvider} +import pl.touk.nussknacker.ui.process.repository.PeriodicProcessesRepository + +class RepositoryBasedPeriodicProcessesManagerProvider( + periodicProcessesRepository: PeriodicProcessesRepository, +) extends PeriodicProcessesManagerProvider { + + override def provide( + deploymentManagerName: String, + processingType: String + ): PeriodicProcessesManager = { + new RepositoryBasedPeriodicProcessesManager( + deploymentManagerName, + processingType, + periodicProcessesRepository + ) + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala similarity index 70% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala rename to designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala index 267f6316fbf..27899913517 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/PeriodicProcessesRepository.scala @@ -1,16 +1,21 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.ui.process.repository +import _root_.db.util.DBIOActionInstances import cats.Monad import com.github.tminglei.slickpg.ExPostgresProfile import com.typesafe.scalalogging.LazyLogging import io.circe.parser.decode +import io.circe.syntax.EncoderOps import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.ui.db.entity._ import slick.dbio.{DBIOAction, Effect, NoStream} import slick.jdbc.PostgresProfile.api._ import slick.jdbc.{JdbcBackend, JdbcProfile} @@ -23,9 +28,9 @@ object PeriodicProcessesRepository { def createPeriodicProcessDeployment( processEntity: PeriodicProcessEntity, - processDeploymentEntity: PeriodicProcessDeploymentEntity - ): PeriodicProcessDeployment[CanonicalProcess] = { - val process = createPeriodicProcessWithJson(processEntity) + processDeploymentEntity: PeriodicProcessDeploymentEntity, + ): PeriodicProcessDeployment = { + val process = createPeriodicProcess(processEntity) PeriodicProcessDeployment( processDeploymentEntity.id, process, @@ -48,36 +53,15 @@ object PeriodicProcessesRepository { ) } - def createPeriodicProcessWithJson(processEntity: PeriodicProcessEntity): PeriodicProcess[CanonicalProcess] = { + def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { val processVersion = createProcessVersion(processEntity) val scheduleProperty = prepareScheduleProperty(processEntity) PeriodicProcess( processEntity.id, - model.DeploymentWithJarData( + DeploymentWithRuntimeParams( processVersion = processVersion, inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, - jarFileName = processEntity.jarFileName, - process = processEntity.processJson.getOrElse( - throw new IllegalArgumentException("Missing required scenario json in processEntity") - ) - ), - scheduleProperty, - processEntity.active, - processEntity.createdAt, - processEntity.processActionId - ) - } - - def createPeriodicProcessWithoutJson(processEntity: PeriodicProcessEntity): PeriodicProcess[Unit] = { - val processVersion = createProcessVersion(processEntity) - val scheduleProperty = prepareScheduleProperty(processEntity) - PeriodicProcess( - processEntity.id, - model.DeploymentWithJarData( - processVersion = processVersion, - inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, - jarFileName = processEntity.jarFileName, - process = () + runtimeParams = processEntity.runtimeParams, ), scheduleProperty, processEntity.active, @@ -89,7 +73,7 @@ object PeriodicProcessesRepository { private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { val scheduleProperty = decode[ScheduleProperty](processEntity.scheduleProperty) .fold(e => throw new IllegalArgumentException(e), identity) - scheduleProperty + toApi(scheduleProperty) } private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { @@ -117,33 +101,35 @@ trait PeriodicProcessesRepository { ): Action[SchedulesState] def create( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - scheduleProperty: ScheduleProperty, - processActionId: ProcessActionId - ): Action[PeriodicProcess[CanonicalProcess]] + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] - def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] - def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, ): Action[SchedulesState] - def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[CanonicalProcess]] - - def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] + def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] @@ -163,7 +149,12 @@ trait PeriodicProcessesRepository { scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[CanonicalProcess]] + ): Action[PeriodicProcessDeployment] + + def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId, + ): Action[Option[CanonicalProcess]] } @@ -171,14 +162,14 @@ class SlickPeriodicProcessesRepository( db: JdbcBackend.DatabaseDef, override val profile: JdbcProfile, clock: Clock, - processingType: String )(implicit ec: ExecutionContext) extends PeriodicProcessesRepository with PeriodicProcessesTableFactory with PeriodicProcessDeploymentsTableFactory + with ProcessVersionEntityFactory + with ProcessEntityFactory with LazyLogging { - import io.circe.syntax._ import pl.touk.nussknacker.engine.util.Implicits._ type Action[T] = DBIOActionInstances.DB[T] @@ -190,7 +181,7 @@ class SlickPeriodicProcessesRepository( override def getSchedulesState( scenarioName: ProcessName ): Action[SchedulesState] = { - PeriodicProcessesWithoutJson + PeriodicProcesses .filter(_.processName === scenarioName) .join(PeriodicProcessDeployments) .on(_.id === _.periodicProcessId) @@ -199,61 +190,66 @@ class SlickPeriodicProcessesRepository( } override def create( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - scheduleProperty: ScheduleProperty, - processActionId: ProcessActionId - ): Action[PeriodicProcess[CanonicalProcess]] = { + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, + processActionId: ProcessActionId, + processingType: String, + ): Action[PeriodicProcess] = { val processEntity = PeriodicProcessEntity( id = PeriodicProcessId(-1), - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithRuntimeParams.processVersion.processName, + processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - processJson = Some(deploymentWithJarData.process), - inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, - jarFileName = deploymentWithJarData.jarFileName, - scheduleProperty = scheduleProperty.asJson.noSpaces, + inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + runtimeParams = deploymentWithRuntimeParams.runtimeParams, + scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = now(), Some(processActionId) ) - ((PeriodicProcessesWithJson returning PeriodicProcessesWithJson into ((_, id) => id)) += processEntity) - .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + ((PeriodicProcesses returning PeriodicProcesses into ((_, id) => id)) += processEntity) + .map(PeriodicProcessesRepository.createPeriodicProcess) } private def now(): LocalDateTime = LocalDateTime.now(clock) - override def findToBeDeployed: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = - activePeriodicProcessWithDeploymentQuery + override def findToBeDeployed(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => d.runAt <= now() && d.status === (PeriodicProcessDeploymentStatus.Scheduled: PeriodicProcessDeploymentStatus) } - .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) + ) - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = - activePeriodicProcessWithDeploymentQuery + override def findToBeRetried(processingType: String): Action[Seq[PeriodicProcessDeployment]] = findProcesses( + activePeriodicProcessWithDeploymentQuery(processingType) .filter { case (_, d) => d.nextRetryAt <= now() && d.status === (PeriodicProcessDeploymentStatus.RetryingDeploy: PeriodicProcessDeploymentStatus) } - .result - .map(_.map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled)) - - override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment[CanonicalProcess]] = { - (PeriodicProcessesWithJson join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) - .filter { case (_, deployment) => deployment.id === id } - .result - .head - .map((PeriodicProcessesRepository.createPeriodicProcessDeployment _).tupled) + ) + + private def findProcesses( + query: Query[ + (PeriodicProcessesTable, PeriodicProcessDeploymentsTable), + (PeriodicProcessEntity, PeriodicProcessDeploymentEntity), + Seq + ] + ) = { + query.result + .map(_.map { case (periodicProcess, periodicDeployment) => + PeriodicProcessesRepository.createPeriodicProcessDeployment( + periodicProcess, + periodicDeployment, + ) + }) } - override def findProcessData(processName: ProcessName): Action[Seq[PeriodicProcess[CanonicalProcess]]] = { - PeriodicProcessesWithJson - .filter(p => p.active === true && p.processName === processName) - .result - .map(_.map(PeriodicProcessesRepository.createPeriodicProcessWithJson)) - } + override def findProcessData(id: PeriodicProcessDeploymentId): Action[PeriodicProcessDeployment] = + findProcesses( + (PeriodicProcesses join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) + .filter { case (_, deployment) => deployment.id === id } + ).map(_.head) override def markDeployed(id: PeriodicProcessDeploymentId): Action[Unit] = { val q = for { @@ -296,9 +292,10 @@ class SlickPeriodicProcessesRepository( } override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + processingType: String, ): Action[SchedulesState] = { - val processesHavingDeploymentsWithMatchingStatus = PeriodicProcessesWithoutJson.filter(p => + val processesHavingDeploymentsWithMatchingStatus = PeriodicProcesses.filter(p => p.active && PeriodicProcessDeployments .filter(d => d.periodicProcessId === p.id && d.status.inSet(expectedDeploymentStatuses)) @@ -306,33 +303,37 @@ class SlickPeriodicProcessesRepository( ) getLatestDeploymentsForEachSchedule( processesHavingDeploymentsWithMatchingStatus, - deploymentsPerScheduleMaxCount = 1 + deploymentsPerScheduleMaxCount = 1, + processingType = processingType, ) } override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { - val activeProcessesQuery = PeriodicProcessesWithoutJson.filter(p => p.processName === processName && p.active) - getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount) + val activeProcessesQuery = PeriodicProcesses.filter(p => p.processName === processName && p.active) + getLatestDeploymentsForEachSchedule(activeProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { - val filteredProcessesQuery = PeriodicProcessesWithoutJson + val filteredProcessesQuery = PeriodicProcesses .filter(p => p.processName === processName && !p.active) .sortBy(_.createdAt.desc) .take(inactiveProcessesMaxCount) - getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount) + getLatestDeploymentsForEachSchedule(filteredProcessesQuery, deploymentsPerScheduleMaxCount, processingType) } private def getLatestDeploymentsForEachSchedule( periodicProcessesQuery: Query[PeriodicProcessesTable, PeriodicProcessEntity, Seq], - deploymentsPerScheduleMaxCount: Int + deploymentsPerScheduleMaxCount: Int, + processingType: String, ): Action[SchedulesState] = { val filteredPeriodicProcessQuery = periodicProcessesQuery.filter(p => p.processingType === processingType) val latestDeploymentsForSchedules = profile match { @@ -421,7 +422,7 @@ class SlickPeriodicProcessesRepository( scheduleName: ScheduleName, runAt: LocalDateTime, deployMaxRetries: Int - ): Action[PeriodicProcessDeployment[CanonicalProcess]] = { + ): Action[PeriodicProcessDeployment] = { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(-1), periodicProcessId = id, @@ -441,14 +442,25 @@ class SlickPeriodicProcessesRepository( override def markInactive(processId: PeriodicProcessId): Action[Unit] = { val q = for { - p <- PeriodicProcessesWithoutJson if p.id === processId + p <- PeriodicProcesses if p.id === processId } yield p.active val update = q.update(false) update.map(_ => ()) } - private def activePeriodicProcessWithDeploymentQuery = { - (PeriodicProcessesWithJson.filter(p => p.active === true && p.processingType === processingType) + def fetchCanonicalProcess(processName: ProcessName, versionId: VersionId): Action[Option[CanonicalProcess]] = { + processesTable + .filter(_.name === processName) + .join(processVersionsTable) + .on((process, version) => process.id === version.processId) + .filter { case (_, version) => version.id === versionId } + .result + .headOption + .map(_.flatMap(_._2.json)) + } + + private def activePeriodicProcessWithDeploymentQuery(processingType: String) = { + (PeriodicProcesses.filter(p => p.active === true && p.processingType === processingType) join PeriodicProcessDeployments on (_.id === _.periodicProcessId)) } @@ -458,8 +470,8 @@ class SlickPeriodicProcessesRepository( .map { case (process, deployment) => val scheduleId = ScheduleId(process.id, ScheduleName(deployment.scheduleName)) val scheduleDataWithoutDeployment = - (scheduleId, PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process)) - val scheduleDeployment = ScheduleDeploymentData(deployment) + (scheduleId, PeriodicProcessesRepository.createPeriodicProcess(process)) + val scheduleDeployment = scheduleDeploymentData(deployment) (scheduleDataWithoutDeployment, scheduleDeployment) } .toList @@ -472,26 +484,16 @@ class SlickPeriodicProcessesRepository( ) } -} - -//Copied from designer/server. -object DBIOActionInstances { - - type DB[A] = DBIOAction[A, NoStream, Effect.All] - - implicit def dbMonad(implicit ec: ExecutionContext): Monad[DB] = new Monad[DB] { - - override def pure[A](x: A) = DBIO.successful(x) - - override def flatMap[A, B](fa: DB[A])(f: (A) => DB[B]) = fa.flatMap(f) - - // this is *not* tail recursive - override def tailRecM[A, B](a: A)(f: (A) => DB[Either[A, B]]): DB[B] = - f(a).flatMap { - case Right(r) => pure(r) - case Left(l) => tailRecM(l)(f) - } - + private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + ScheduleDeploymentData( + deployment.id, + deployment.createdAt, + deployment.runAt, + deployment.deployedAt, + deployment.retriesLeft, + deployment.nextRetryAt, + PeriodicProcessesRepository.createPeriodicDeploymentState(deployment) + ) } } diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala index 287f44bceca..df84e67e381 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/process/repository/ScenarioActionRepository.scala @@ -7,7 +7,7 @@ import pl.touk.nussknacker.engine.api.Comment import pl.touk.nussknacker.engine.api.deployment.ProcessActionState.ProcessActionState import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType, VersionId} -import pl.touk.nussknacker.engine.management.periodic.InstantBatchCustomAction +import pl.touk.nussknacker.engine.common.periodic.InstantBatchCustomAction import pl.touk.nussknacker.engine.util.Implicits.RichScalaMap import pl.touk.nussknacker.ui.app.BuildInfo import pl.touk.nussknacker.ui.db.entity.{ diff --git a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala index 822727dc6d4..2b3c183d9cd 100644 --- a/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala +++ b/designer/server/src/main/scala/pl/touk/nussknacker/ui/server/AkkaHttpBasedRouteProvider.scala @@ -64,6 +64,10 @@ import pl.touk.nussknacker.ui.process.newdeployment.synchronize.{ DeploymentsStatusesSynchronizer } import pl.touk.nussknacker.ui.process.newdeployment.{DeploymentRepository, DeploymentService} +import pl.touk.nussknacker.ui.process.periodic.{ + RepositoryBasedPeriodicProcessesManager, + RepositoryBasedPeriodicProcessesManagerProvider +} import pl.touk.nussknacker.ui.process.processingtype.ProcessingTypeData import pl.touk.nussknacker.ui.process.processingtype.loader.ProcessingTypeDataLoader import pl.touk.nussknacker.ui.process.processingtype.provider.ReloadableProcessingTypeDataProvider @@ -123,15 +127,17 @@ class AkkaHttpBasedRouteProvider( featureTogglesConfig = FeatureTogglesConfig.create(resolvedConfig) _ = logger.info(s"Designer config loaded: \nfeatureTogglesConfig: $featureTogglesConfig") countsReporter <- createCountsReporter(featureTogglesConfig, environment, sttpBackend) - actionServiceSupplier = new DelayedInitActionServiceSupplier - additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedConfig, sttpBackend) - deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) - scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) - dbioRunner = DBIOActionRunner(dbRef) + actionServiceSupplier = new DelayedInitActionServiceSupplier + additionalUIConfigProvider = createAdditionalUIConfigProvider(resolvedConfig, sttpBackend) + deploymentRepository = new DeploymentRepository(dbRef, Clock.systemDefaultZone()) + scenarioActivityRepository = DbScenarioActivityRepository.create(dbRef, designerClock) + periodicProcessesRepository = new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, designerClock) + dbioRunner = DBIOActionRunner(dbRef) processingTypeDataProvider <- prepareProcessingTypeDataReload( additionalUIConfigProvider, actionServiceSupplier, scenarioActivityRepository, + periodicProcessesRepository, dbioRunner, sttpBackend, featureTogglesConfig, @@ -698,6 +704,7 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider: AdditionalUIConfigProvider, actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, + periodicProcessesRepository: PeriodicProcessesRepository, dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], featureTogglesConfig: FeatureTogglesConfig @@ -716,6 +723,7 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider, actionServiceProvider, scenarioActivityRepository, + periodicProcessesRepository, dbioActionRunner, sttpBackend, _ @@ -732,6 +740,7 @@ class AkkaHttpBasedRouteProvider( additionalUIConfigProvider: AdditionalUIConfigProvider, actionServiceProvider: Supplier[ActionService], scenarioActivityRepository: ScenarioActivityRepository, + periodicProcessesRepository: PeriodicProcessesRepository, dbioActionRunner: DBIOActionRunner, sttpBackend: SttpBackend[Future, Any], processingType: ProcessingType @@ -747,6 +756,7 @@ class AkkaHttpBasedRouteProvider( scenarioActivityRepository, dbioActionRunner, ), + new RepositoryBasedPeriodicProcessesManagerProvider(periodicProcessesRepository), system.dispatcher, system, sttpBackend, diff --git a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala index bc9c38f453f..7196d3bf02d 100644 --- a/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala +++ b/designer/server/src/test/scala/db/migration/V1_057__MigrateActionsAndCommentsToScenarioActivities.scala @@ -8,12 +8,11 @@ import db.migration.V1_057__MigrateActionsAndCommentsToScenarioActivitiesDefinit import io.circe.syntax.EncoderOps import org.scalatest.freespec.AnyFreeSpecLike import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.api.deployment.ScenarioComment.WithContent import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessAdditionalFields, RequestResponseMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.InstantBatchCustomAction +import pl.touk.nussknacker.engine.common.periodic.InstantBatchCustomAction import pl.touk.nussknacker.restmodel.component.ScenarioComponentsUsages import pl.touk.nussknacker.test.base.db.WithHsqlDbTesting import pl.touk.nussknacker.test.base.it.NuItTest diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala index fbc1af4f450..adf9fd3b3cf 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/base/db/DbTesting.scala @@ -93,6 +93,8 @@ trait DbTesting extends BeforeAndAfterEach with BeforeAndAfterAll { session.prepareStatement("""delete from "environments"""").execute() session.prepareStatement("""delete from "processes"""").execute() session.prepareStatement("""delete from "fingerprints"""").execute() + session.prepareStatement("""delete from "periodic_processes"""").execute() + session.prepareStatement("""delete from "periodic_process_deployments"""").execute() } } diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala index dcd25483e6d..605749a8ee9 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/mock/MockDeploymentManager.scala @@ -13,6 +13,10 @@ import pl.touk.nussknacker.engine.api.definition.{ StringParameterEditor } import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.{ + NoOpPeriodicProcessesManagerProvider, + PeriodicProcessesManagerProvider +} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.api.{ProcessVersion, StreamMetaData} @@ -44,6 +48,7 @@ class MockDeploymentManager( new ProcessingTypeDeployedScenariosProviderStub(List.empty), actionService: ProcessingTypeActionService = new ProcessingTypeActionServiceStub, scenarioActivityManager: ScenarioActivityManager = NoOpScenarioActivityManager, + periodicProcessesManagerProvider: PeriodicProcessesManagerProvider = NoOpPeriodicProcessesManagerProvider, ) extends FlinkDeploymentManager( ModelData( ProcessingTypeConfig.read(ConfigWithScalaVersion.StreamingProcessTypeConfig), @@ -53,6 +58,7 @@ class MockDeploymentManager( deployedScenariosProvider, actionService, scenarioActivityManager, + periodicProcessesManagerProvider, ExecutionContext.global, ActorSystem("MockDeploymentManager"), SttpBackendStub.asynchronousFuture diff --git a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala index c95746e8536..3a8779b0516 100644 --- a/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/test/utils/domain/TestFactory.scala @@ -8,6 +8,10 @@ import com.typesafe.config.ConfigFactory import db.util.DBIOActionInstances._ import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId, ProcessingMode} import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue +import pl.touk.nussknacker.engine.api.deployment.periodic.{ + NoOpPeriodicProcessesManager, + NoOpPeriodicProcessesManagerProvider +} import pl.touk.nussknacker.engine.api.deployment.{ NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, @@ -139,6 +143,7 @@ object TestFactory { new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, actorSystem.dispatcher, actorSystem, SttpBackendStub.asynchronousFuture diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala similarity index 81% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala rename to designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala index 889b47a577f..fe040de31c9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceIntegrationTest.scala +++ b/designer/server/src/test/scala/pl/touk/nussknacker/ui/process/periodic/PeriodicProcessServiceIntegrationTest.scala @@ -1,11 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.ui.process.periodic +import _root_.db.util.DBIOActionInstances.DB import com.cronutils.builder.CronBuilder import com.cronutils.model.CronType import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.field.expression.FieldExpressionFactory.{on, questionMark} -import com.dimafeng.testcontainers.{ForAllTestContainer, PostgreSQLContainer} -import com.typesafe.config.{Config, ConfigFactory} import com.typesafe.scalalogging.LazyLogging import org.scalatest.LoneElement._ import org.scalatest.OptionValues @@ -14,37 +13,36 @@ import org.scalatest.exceptions.TestFailedException import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Seconds, Span} -import org.testcontainers.utility.DockerImageName +import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus -import pl.touk.nussknacker.engine.api.deployment.{ - DataFreshnessPolicy, - ProcessActionId, - ProcessingTypeActionServiceStub, - ScenarioActivity, - ScenarioId, - ScenarioUser, - ScenarioVersionId, - ScheduledExecutionStatus, - UserName -} -import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.db.{DbInitializer, SlickPeriodicProcessesRepository} -import pl.touk.nussknacker.engine.management.periodic.model._ -import pl.touk.nussknacker.engine.management.periodic.service._ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.management.periodic.flink.{DeploymentManagerStub, PeriodicDeploymentHandlerStub} import pl.touk.nussknacker.test.PatientScalaFutures -import slick.jdbc -import slick.jdbc.{JdbcBackend, JdbcProfile} +import pl.touk.nussknacker.test.base.db.WithPostgresDbTesting +import pl.touk.nussknacker.test.base.it.WithClock +import pl.touk.nussknacker.test.utils.domain.TestFactory.newWriteProcessRepository +import pl.touk.nussknacker.test.utils.scalas.DBIOActionValues +import pl.touk.nussknacker.ui.db.DbRef +import pl.touk.nussknacker.ui.process.repository.ProcessRepository.CreateProcessAction +import pl.touk.nussknacker.ui.process.repository.{ + DBIOActionRunner, + DBProcessRepository, + SlickPeriodicProcessesRepository +} +import pl.touk.nussknacker.ui.security.api.AdminUser import java.time._ import java.time.temporal.ChronoUnit import java.util.UUID import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future -import scala.jdk.CollectionConverters._ //Integration test with both in-memory hsql and postgres from test containers class PeriodicProcessServiceIntegrationTest @@ -53,23 +51,21 @@ class PeriodicProcessServiceIntegrationTest with OptionValues with ScalaFutures with PatientScalaFutures - with ForAllTestContainer - with LazyLogging { - - override val container: PostgreSQLContainer = PostgreSQLContainer(DockerImageName.parse("postgres:11.2")) + with WithPostgresDbTesting + with LazyLogging + with WithClock + with DBIOActionValues { import scala.concurrent.ExecutionContext.Implicits.global + override protected def dbioRunner: DBIOActionRunner = new DBIOActionRunner(testDbRef) + implicit val freshnessPolicy: DataFreshnessPolicy = DataFreshnessPolicy.Fresh private val processingType = "testProcessingType" private val processName = ProcessName("test") - private val processIdWithName = ProcessIdWithName(ProcessId(1), processName) - - private val sampleProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) - private val startTime = Instant.parse("2021-04-06T13:18:00Z") // we truncate to millis, as HSQL stores with that precision... @@ -84,55 +80,34 @@ class PeriodicProcessServiceIntegrationTest deploymentRetryConfig: DeploymentRetryConfig = DeploymentRetryConfig(), executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig() )(testCode: Fixture => Any): Unit = { - val postgresConfig = ConfigFactory.parseMap( - Map( - "user" -> container.username, - "password" -> container.password, - "url" -> container.jdbcUrl, - "driver" -> "org.postgresql.Driver", - "schema" -> UUID.randomUUID().toString - ).asJava - ) - - val hsqlConfig = ConfigFactory.parseMap( - Map( - "url" -> s"jdbc:hsqldb:mem:periodic-${UUID.randomUUID().toString};sql.syntax_ora=true", - "user" -> "SA", - "password" -> "" - ).asJava - ) - - def runTestCodeWithDbConfig(config: Config) = { - val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(config) - try { - testCode(new Fixture(db, dbProfile, deploymentRetryConfig, executionConfig)) - } finally { - db.close() - } + def runTestCodeWithDbConfig = { + testCode(new Fixture(testDbRef, deploymentRetryConfig, executionConfig)) } - logger.debug("Running test with hsql") - runTestCodeWithDbConfig(hsqlConfig) - logger.debug("Running test with postgres") - runTestCodeWithDbConfig(postgresConfig) + logger.debug("Running test with database") + runTestCodeWithDbConfig } class Fixture( - db: JdbcBackend.DatabaseDef, - dbProfile: JdbcProfile, + dbRef: DbRef, deploymentRetryConfig: DeploymentRetryConfig, executionConfig: PeriodicExecutionConfig ) { val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val events = new ArrayBuffer[PeriodicProcessEvent]() var failListener = false - def periodicProcessService(currentTime: Instant, processingType: String = processingType) = + def periodicProcessService( + currentTime: Instant, + deploymentManagerName: String = "testPeriodicDeploymentManager", + processingType: String = processingType + ) = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = - new SlickPeriodicProcessesRepository(db, dbProfile, fixedClock(currentTime), processingType), + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = new RepositoryBasedPeriodicProcessesManagerProvider( + new SlickPeriodicProcessesRepository(dbRef.db, dbRef.profile, fixedClock(currentTime)) + ).provide(deploymentManagerName, processingType), periodicProcessListener = new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { @@ -147,8 +122,26 @@ class PeriodicProcessServiceIntegrationTest processConfigEnricher = ProcessConfigEnricher.identity, clock = fixedClock(currentTime), new ProcessingTypeActionServiceStub, - Map.empty + Map.empty, + ) + + def writeProcessRepository: DBProcessRepository = newWriteProcessRepository(dbRef, clock) + + def prepareProcess(processName: ProcessName): DB[ProcessIdWithName] = { + val canonicalProcess = CanonicalProcess(MetaData(processName.value, StreamMetaData()), Nil) + val action = CreateProcessAction( + processName = processName, + category = "Category1", + canonicalProcess = canonicalProcess, + processingType = "streaming", + isFragment = false, + forwardedUserName = None ) + writeProcessRepository + .saveNewProcess(action)(AdminUser("artificialTestAdmin", "artificialTestAdmin")) + .map(_.value.processId) + .map(ProcessIdWithName(_, processName)) + } } @@ -165,36 +158,34 @@ class PeriodicProcessServiceIntegrationTest def otherProcessingTypeService = f.periodicProcessService(currentTime, processingType = "other") val otherProcessName = ProcessName("other") + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue service .schedule( cronEvery30Minutes, ProcessVersion.empty.copy(processName = every30MinutesProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue service .schedule( cronEvery4Hours, ProcessVersion.empty.copy(processName = every4HoursProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue otherProcessingTypeService .schedule( cronEveryHour, ProcessVersion.empty.copy(processName = otherProcessName), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue @@ -257,7 +248,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -277,15 +268,17 @@ class PeriodicProcessServiceIntegrationTest ) { f => val timeToTriggerCheck = startTime.plus(2, ChronoUnit.HOURS) var currentTime = startTime - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) def service = f.periodicProcessService(currentTime) + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -306,7 +299,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -331,6 +324,9 @@ class PeriodicProcessServiceIntegrationTest val scheduleMinute5 = "scheduleMinute5" val scheduleMinute10 = "scheduleMinute10" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -339,9 +335,8 @@ class PeriodicProcessServiceIntegrationTest scheduleMinute10 -> CronScheduleProperty("0 10 * * * ?") ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -355,8 +350,7 @@ class PeriodicProcessServiceIntegrationTest ) ), ProcessVersion.empty.copy(processName = ProcessName("other")), - sampleProcess, - randomProcessActionId + randomProcessActionId, ) .futureValue @@ -396,6 +390,9 @@ class PeriodicProcessServiceIntegrationTest val firstSchedule = "schedule1" val secondSchedule = "schedule2" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -404,9 +401,8 @@ class PeriodicProcessServiceIntegrationTest secondSchedule -> CronScheduleProperty("0 5 * * * ?") ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -436,7 +432,7 @@ class PeriodicProcessServiceIntegrationTest } firstActivity shouldBe ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -467,6 +463,9 @@ class PeriodicProcessServiceIntegrationTest val schedule1 = "schedule1" val schedule2 = "schedule2" + + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( MultipleScheduleProperty( @@ -475,9 +474,8 @@ class PeriodicProcessServiceIntegrationTest schedule2 -> CronScheduleProperty(convertDateToCron(localTime(timeToTriggerSchedule2))) ) ), - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue @@ -554,7 +552,7 @@ class PeriodicProcessServiceIntegrationTest val secondActivity = activities(1).asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, @@ -567,7 +565,7 @@ class PeriodicProcessServiceIntegrationTest nextRetryAt = None ), ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = secondActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = secondActivity.date, @@ -589,19 +587,20 @@ class PeriodicProcessServiceIntegrationTest def service = f.periodicProcessService(currentTime) - def tryWithFailedListener[T](action: () => Future[T]): T = { + def tryWithFailedListener[T](action: () => Future[T]): Unit = { f.failListener = true - intercept[TestFailedException](action().futureValue).getCause shouldBe a[PeriodicProcessException] + val exception = intercept[TestFailedException](action().futureValue) + exception.getCause shouldBe a[PeriodicProcessException] f.failListener = false - action().futureValue } + val processIdWithName = f.prepareProcess(processName).dbioActionValues + tryWithFailedListener { () => service.schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) } @@ -623,15 +622,16 @@ class PeriodicProcessServiceIntegrationTest val timeToTriggerCheck = startTime.plus(1, ChronoUnit.HOURS) var currentTime = startTime - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) def service = f.periodicProcessService(currentTime) + val processIdWithName = f.prepareProcess(processName).dbioActionValues + service .schedule( cronEveryHour, - ProcessVersion.empty.copy(processName = processName), - sampleProcess, - randomProcessActionId + ProcessVersion(VersionId(1), processIdWithName.name, processIdWithName.id, List.empty, "testUser", None), + randomProcessActionId, ) .futureValue currentTime = timeToTriggerCheck @@ -652,7 +652,7 @@ class PeriodicProcessServiceIntegrationTest val firstActivity = activities.head.asInstanceOf[ScenarioActivity.PerformedScheduledExecution] activities shouldBe List( ScenarioActivity.PerformedScheduledExecution( - scenarioId = ScenarioId(1), + scenarioId = ScenarioId(processIdWithName.id.value), scenarioActivityId = firstActivity.scenarioActivityId, user = ScenarioUser(None, UserName("Nussknacker"), None, None), date = firstActivity.date, diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala similarity index 74% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala index 091a1cc5052..ba383b46eab 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/DeploymentActor.scala @@ -1,14 +1,13 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{Actor, Props, Timers} import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.{ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.{ CheckToBeDeployed, DeploymentCompleted, WaitingForDeployment } -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment import scala.concurrent.Future import scala.concurrent.duration._ @@ -20,24 +19,24 @@ object DeploymentActor { props(service.findToBeDeployed, service.deploy, interval) } - private[periodic] def props( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]], - deploy: PeriodicProcessDeployment[CanonicalProcess] => Future[Unit], + private[engine] def props( + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) = { Props(new DeploymentActor(findToBeDeployed, deploy, interval)) } - private[periodic] case object CheckToBeDeployed + private[engine] case object CheckToBeDeployed - private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment[CanonicalProcess]]) + private case class WaitingForDeployment(ids: List[PeriodicProcessDeployment]) private case object DeploymentCompleted } class DeploymentActor( - findToBeDeployed: => Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]], - deploy: PeriodicProcessDeployment[CanonicalProcess] => Future[Unit], + findToBeDeployed: => Future[Seq[PeriodicProcessDeployment]], + deploy: PeriodicProcessDeployment => Future[Unit], interval: FiniteDuration ) extends Actor with Timers @@ -73,7 +72,7 @@ class DeploymentActor( } } - private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Receive = { + private def receiveOngoingDeployment(runDetails: PeriodicProcessDeployment): Receive = { case CheckToBeDeployed => logger.debug(s"Still waiting for ${runDetails.display} to be deployed") case DeploymentCompleted => diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala similarity index 96% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala index 3f79c3443fe..db908eb2849 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicBatchConfig.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicBatchConfig.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.typesafe.config.Config diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicCustomActionsProviderFactory.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicCustomActionsProviderFactory.scala similarity index 65% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicCustomActionsProviderFactory.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicCustomActionsProviderFactory.scala index 648d8cb29b3..b1d0258027d 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicCustomActionsProviderFactory.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicCustomActionsProviderFactory.scala @@ -1,23 +1,24 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import pl.touk.nussknacker.engine.api.deployment.DMCustomActionCommand +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, CustomActionResult} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository import scala.concurrent.Future trait PeriodicCustomActionsProviderFactory { def create( - periodicProcessesRepository: PeriodicProcessesRepository, - periodicProcessService: PeriodicProcessService + periodicProcessesManager: PeriodicProcessesManager, + periodicProcessService: PeriodicProcessService, + processingType: String, ): PeriodicCustomActionsProvider } object PeriodicCustomActionsProviderFactory { - def noOp: PeriodicCustomActionsProviderFactory = (_: PeriodicProcessesRepository, _: PeriodicProcessService) => - EmptyPeriodicCustomActionsProvider + def noOp: PeriodicCustomActionsProviderFactory = + (_: PeriodicProcessesManager, _: PeriodicProcessService, _: String) => EmptyPeriodicCustomActionsProvider } trait PeriodicCustomActionsProvider { diff --git a/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala new file mode 100644 index 00000000000..81a0b8bd3f2 --- /dev/null +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentHandler.scala @@ -0,0 +1,26 @@ +package pl.touk.nussknacker.engine.common.periodic + +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} + +import scala.concurrent.Future + +trait PeriodicDeploymentHandler { + + def prepareDeploymentWithRuntimeParams( + processVersion: ProcessVersion, + ): Future[DeploymentWithRuntimeParams] + + def deployWithRuntimeParams( + deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, + ): Future[Option[ExternalDeploymentId]] + + def cleanAfterDeployment( + runtimeParams: RuntimeParams + ): Future[Unit] + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala similarity index 87% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala index dae199ecf46..b027bce3bb3 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManager.scala @@ -1,24 +1,20 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.typesafe.config.Config import com.typesafe.scalalogging.LazyLogging +import pl.touk.nussknacker.engine.DeploymentManagerDependencies import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.FlinkConfig -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.Utils.{createActorWithRetry, runSafely} -import pl.touk.nussknacker.engine.management.periodic.db.{DbInitializer, SlickPeriodicProcessesRepository} -import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic.Utils.{createActorWithRetry, runSafely} +import pl.touk.nussknacker.engine.common.periodic.service.{ AdditionalDeploymentDataProvider, PeriodicProcessListenerFactory, ProcessConfigEnricherFactory } -import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies} -import slick.jdbc -import slick.jdbc.JdbcProfile +import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, ExternalDeploymentId} import java.time.Clock import scala.concurrent.{ExecutionContext, Future} @@ -27,31 +23,26 @@ object PeriodicDeploymentManager { def apply( delegate: DeploymentManager, + periodicDeploymentHandler: PeriodicDeploymentHandler, schedulePropertyExtractorFactory: SchedulePropertyExtractorFactory, processConfigEnricherFactory: ProcessConfigEnricherFactory, periodicBatchConfig: PeriodicBatchConfig, - flinkConfig: FlinkConfig, originalConfig: Config, - modelData: BaseModelData, listenerFactory: PeriodicProcessListenerFactory, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, customActionsProviderFactory: PeriodicCustomActionsProviderFactory, - dependencies: DeploymentManagerDependencies + dependencies: DeploymentManagerDependencies, + periodicProcessesManager: PeriodicProcessesManager, ): PeriodicDeploymentManager = { import dependencies._ - val clock = Clock.systemDefaultZone() - - val (db: jdbc.JdbcBackend.DatabaseDef, dbProfile: JdbcProfile) = DbInitializer.init(periodicBatchConfig.db) - val scheduledProcessesRepository = - new SlickPeriodicProcessesRepository(db, dbProfile, clock, periodicBatchConfig.processingType) - val jarManager = FlinkJarManager(flinkConfig, periodicBatchConfig, modelData) + val clock = Clock.systemDefaultZone() val listener = listenerFactory.create(originalConfig) val processConfigEnricher = processConfigEnricherFactory(originalConfig) val service = new PeriodicProcessService( delegate, - jarManager, - scheduledProcessesRepository, + periodicDeploymentHandler, + periodicProcessesManager, listener, additionalDeploymentDataProvider, periodicBatchConfig.deploymentRetry, @@ -59,7 +50,7 @@ object PeriodicDeploymentManager { processConfigEnricher, clock, dependencies.actionService, - dependencies.configsFromProvider + dependencies.configsFromProvider, ) // These actors have to be created with retries because they can initially fail to create due to taken names, @@ -75,7 +66,11 @@ object PeriodicDeploymentManager { dependencies.actorSystem ) - val customActionsProvider = customActionsProviderFactory.create(scheduledProcessesRepository, service) + val customActionsProvider = customActionsProviderFactory.create( + periodicProcessesManager, + service, + periodicBatchConfig.processingType + ) val toClose = () => { runSafely(listener.close()) @@ -83,24 +78,25 @@ object PeriodicDeploymentManager { // they don't have any internal state, so stopping them non-gracefully is safe runSafely(dependencies.actorSystem.stop(deploymentActor)) runSafely(dependencies.actorSystem.stop(rescheduleFinishedActor)) - runSafely(db.close()) } new PeriodicDeploymentManager( delegate, service, schedulePropertyExtractorFactory(originalConfig), customActionsProvider, + periodicBatchConfig.processingType, toClose ) } } -class PeriodicDeploymentManager private[periodic] ( +class PeriodicDeploymentManager private[engine] ( val delegate: DeploymentManager, service: PeriodicProcessService, schedulePropertyExtractor: SchedulePropertyExtractor, customActionsProvider: PeriodicCustomActionsProvider, + processingType: String, toClose: () => Unit )(implicit val ec: ExecutionContext) extends DeploymentManager @@ -140,7 +136,6 @@ class PeriodicDeploymentManager private[periodic] ( .schedule( scheduleProperty, processVersion, - canonicalProcess, deploymentData.deploymentId.toActionIdOpt.getOrElse( throw new IllegalArgumentException(s"deploymentData.deploymentId should be valid ProcessActionId") ), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala similarity index 75% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala index 849ce88392b..af2620b0efd 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/FlinkPeriodicDeploymentManagerProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicDeploymentManagerProvider.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.data.ValidatedNel import com.typesafe.config.Config @@ -6,11 +6,9 @@ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.api.component.ScenarioPropertyConfig import pl.touk.nussknacker.engine.api.definition.{MandatoryParameterValidator, StringParameterEditor} import pl.touk.nussknacker.engine.api.deployment.DeploymentManager +import pl.touk.nussknacker.engine.common.periodic.cron.CronParameterValidator +import pl.touk.nussknacker.engine.common.periodic.service._ import pl.touk.nussknacker.engine.deployment.EngineSetupName -import pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator -import pl.touk.nussknacker.engine.management.{FlinkConfig, FlinkStreamingDeploymentManagerProvider} -import pl.touk.nussknacker.engine.management.periodic.service._ -import pl.touk.nussknacker.engine.util.config.ConfigEnrichments.RichConfig import pl.touk.nussknacker.engine.{ BaseModelData, DeploymentManagerDependencies, @@ -20,9 +18,11 @@ import pl.touk.nussknacker.engine.{ import scala.concurrent.duration.FiniteDuration -class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider with LazyLogging { - - private val delegate = new FlinkStreamingDeploymentManagerProvider() +abstract class PeriodicDeploymentManagerProvider( + override val name: String, + delegate: DeploymentManagerProvider, +) extends DeploymentManagerProvider + with LazyLogging { private val cronConfig = CronSchedulePropertyExtractor.CronPropertyDefaultName -> ScenarioPropertyConfig( defaultValue = None, @@ -32,7 +32,11 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w hintText = Some("Quartz cron syntax. You can specify multiple schedulers separated by '|'.") ) - override def name: String = "flinkPeriodic" + protected def createPeriodicDeploymentHandler( + modelData: BaseModelData, + dependencies: DeploymentManagerDependencies, + config: Config, + ): PeriodicDeploymentHandler override def createDeploymentManager( modelData: BaseModelData, @@ -40,26 +44,25 @@ class FlinkPeriodicDeploymentManagerProvider extends DeploymentManagerProvider w config: Config, scenarioStateCacheTTL: Option[FiniteDuration] ): ValidatedNel[String, DeploymentManager] = { - logger.info("Creating FlinkPeriodic scenario manager") + logger.info("Creating periodic scenario manager") delegate.createDeploymentManager(modelData, dependencies, config, scenarioStateCacheTTL).map { delegateDeploymentManager => import net.ceedubs.ficus.Ficus._ import net.ceedubs.ficus.readers.ArbitraryTypeReader._ val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") - val flinkConfig = config.rootAs[FlinkConfig] PeriodicDeploymentManager( delegate = delegateDeploymentManager, + periodicDeploymentHandler = createPeriodicDeploymentHandler(modelData, dependencies, config), schedulePropertyExtractorFactory = _ => CronSchedulePropertyExtractor(), processConfigEnricherFactory = ProcessConfigEnricherFactory.noOp, periodicBatchConfig = periodicBatchConfig, - flinkConfig = flinkConfig, originalConfig = config, - modelData = modelData, EmptyPeriodicProcessListenerFactory, DefaultAdditionalDeploymentDataProvider, new WithRunNowPeriodicCustomActionsProviderFactory, - dependencies + dependencies, + dependencies.periodicProcessesManagerProvider.provide(name, periodicBatchConfig.processingType) ) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala similarity index 74% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala index 72a267b3317..9890980319f 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessException.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessException.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic class PeriodicProcessException(message: String, parent: Throwable) extends RuntimeException(message, parent) { def this(message: String) = this(message, null) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala similarity index 83% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala index e346c92ec08..75d974bdff5 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessService.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessService.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.implicits._ import com.typesafe.scalalogging.LazyLogging @@ -10,23 +10,23 @@ import pl.touk.nussknacker.engine.api.component.{ } import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{ DeploymentStatus, EngineStatusesToReschedule, FinishedScheduledExecutionMetadata, MaxDeploymentsStatus, PeriodicProcessStatus } -import pl.touk.nussknacker.engine.management.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ -import pl.touk.nussknacker.engine.management.periodic.service._ +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic.service._ +import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId} import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor import java.time.chrono.ChronoLocalDateTime @@ -37,8 +37,8 @@ import scala.util.control.NonFatal class PeriodicProcessService( delegateDeploymentManager: DeploymentManager, - jarManager: JarManager, - scheduledProcessesRepository: PeriodicProcessesRepository, + periodicDeploymentHandler: PeriodicDeploymentHandler, + periodicProcessesManager: PeriodicProcessesManager, periodicProcessListener: PeriodicProcessListener, additionalDeploymentDataProvider: AdditionalDeploymentDataProvider, deploymentRetryConfig: DeploymentRetryConfig, @@ -46,19 +46,18 @@ class PeriodicProcessService( processConfigEnricher: ProcessConfigEnricher, clock: Clock, actionService: ProcessingTypeActionService, - configsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig] + configsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig], )(implicit ec: ExecutionContext) extends LazyLogging { import cats.syntax.all._ - import scheduledProcessesRepository._ - private type RepositoryAction[T] = scheduledProcessesRepository.Action[T] - private type Callback = () => Future[Unit] - private type NeedsReschedule = Boolean - private implicit class WithCallbacksSeq(result: RepositoryAction[List[Callback]]) { + private type Callback = () => Future[Unit] + private type NeedsReschedule = Boolean + + private implicit class WithCallbacksSeq(result: Future[List[Callback]]) { def runWithCallbacks: Future[Unit] = - result.run.flatMap(callbacks => Future.sequence(callbacks.map(_()))).map(_ => ()) + result.flatMap(callbacks => Future.sequence(callbacks.map(_()))).map(_ => ()) } private val emptyCallback: Callback = () => Future.successful(()) @@ -68,7 +67,7 @@ class PeriodicProcessService( def getScenarioActivitiesSpecificToPeriodicProcess( processIdWithName: ProcessIdWithName ): Future[List[ScenarioActivity]] = for { - schedulesState <- scheduledProcessesRepository.getSchedulesState(processIdWithName.name).run + schedulesState <- periodicProcessesManager.getSchedulesState(processIdWithName.name) groupedByProcess = schedulesState.groupedByPeriodicProcess deployments = groupedByProcess.flatMap(_.deployments) deploymentsWithStatuses = deployments.flatMap(d => scheduledExecutionStatusAndDateFinished(d).map((d, _))) @@ -92,15 +91,12 @@ class PeriodicProcessService( def schedule( schedule: ScheduleProperty, processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess, processActionId: ProcessActionId, beforeSchedule: => Future[Unit] = Future.unit ): Future[Unit] = { prepareInitialScheduleDates(schedule) match { case Right(scheduleDates) => - beforeSchedule.flatMap(_ => - scheduleWithInitialDates(schedule, processVersion, canonicalProcess, scheduleDates, processActionId) - ) + beforeSchedule.flatMap(_ => scheduleWithInitialDates(schedule, processVersion, scheduleDates, processActionId)) case Left(error) => Future.failed(error) } @@ -129,58 +125,62 @@ class PeriodicProcessService( private def scheduleWithInitialDates( scheduleProperty: ScheduleProperty, processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - processActionId: ProcessActionId + processActionId: ProcessActionId, ): Future[Unit] = { logger.info("Scheduling periodic scenario: {} on {}", processVersion, scheduleDates) for { - deploymentWithJarData <- jarManager.prepareDeploymentWithJar(processVersion, canonicalProcess) + deploymentWithJarData <- periodicDeploymentHandler.prepareDeploymentWithRuntimeParams( + processVersion, + ) enrichedProcessConfig <- processConfigEnricher.onInitialSchedule( ProcessConfigEnricher.InitialScheduleData( - deploymentWithJarData.process, deploymentWithJarData.inputConfigDuringExecutionJson ) ) enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = enrichedProcessConfig.inputConfigDuringExecutionJson ) - _ <- initialSchedule(scheduleProperty, scheduleDates, enrichedDeploymentWithJarData, processActionId) + _ <- initialSchedule( + scheduleProperty, + scheduleDates, + enrichedDeploymentWithJarData, + processActionId, + ) } yield () } private def initialSchedule( scheduleMap: ScheduleProperty, scheduleDates: List[(ScheduleName, Option[LocalDateTime])], - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - processActionId: ProcessActionId + deploymentWithJarData: DeploymentWithRuntimeParams, + processActionId: ProcessActionId, ): Future[Unit] = { - scheduledProcessesRepository - .create(deploymentWithJarData, scheduleMap, processActionId) + periodicProcessesManager + .create(deploymentWithJarData, toApi(scheduleMap), processActionId) .flatMap { process => scheduleDates.collect { case (name, Some(date)) => - scheduledProcessesRepository + periodicProcessesManager .schedule(process.id, name, date, deploymentRetryConfig.deployMaxRetries) .flatMap { data => handleEvent(ScheduledEvent(data, firstSchedule = true)) } case (name, None) => logger.warn(s"Schedule $name does not have date to schedule") - monad.pure(()) + Future.successful(()) }.sequence } - .run .map(_ => ()) } - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { for { - toBeDeployed <- scheduledProcessesRepository.findToBeDeployed.run.flatMap { toDeployList => + toBeDeployed <- periodicProcessesManager.findToBeDeployed.flatMap { toDeployList => Future.sequence(toDeployList.map(checkIfNotRunning)).map(_.flatten) } // We retry scenarios that failed on deployment. Failure recovery of running scenarios should be handled by Flink's restart strategy - toBeRetried <- scheduledProcessesRepository.findToBeRetried.run + toBeRetried <- periodicProcessesManager.findToBeRetried // We don't block scheduled deployments by retries } yield toBeDeployed.sortBy(d => (d.runAt, d.createdAt)) ++ toBeRetried.sortBy(d => (d.nextRetryAt, d.createdAt)) } @@ -188,8 +188,8 @@ class PeriodicProcessService( // Currently we don't allow simultaneous runs of one scenario - only sequential, so if other schedule kicks in, it'll have to wait // TODO: we show allow to deploy scenarios with different scheduleName to be deployed simultaneous private def checkIfNotRunning( - toDeploy: PeriodicProcessDeployment[CanonicalProcess] - ): Future[Option[PeriodicProcessDeployment[CanonicalProcess]]] = { + toDeploy: PeriodicProcessDeployment + ): Future[Option[PeriodicProcessDeployment]] = { delegateDeploymentManager .getProcessStates(toDeploy.periodicProcess.processVersion.processName)(DataFreshnessPolicy.Fresh) .map( @@ -211,7 +211,7 @@ class PeriodicProcessService( schedules.groupedByPeriodicProcess .collect { case processScheduleData - if processScheduleData.existsDeployment(d => needRescheduleDeploymentIds.contains(d.id)) => + if processScheduleData.deployments.exists(d => needRescheduleDeploymentIds.contains(d.id)) => reschedule(processScheduleData, needRescheduleDeploymentIds) } .sequence @@ -219,11 +219,10 @@ class PeriodicProcessService( } for { - schedules <- scheduledProcessesRepository + schedules <- periodicProcessesManager .findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - Set(PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.FailedOnDeploy) + Set(PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.FailedOnDeploy), ) - .run // we handle each job separately, if we fail at some point, we will continue on next handleFinished run _ <- Future.sequence(schedules.groupByProcessName.toList.map(handleSingleProcess _ tupled)) } yield () @@ -244,7 +243,7 @@ class PeriodicProcessService( }) needRescheduleDeployments <- Future .sequence(scheduleDeploymentsWithStatus.map { case (deploymentData, statusOpt) => - synchronizeDeploymentState(deploymentData, statusOpt).run.map { needReschedule => + synchronizeDeploymentState(deploymentData, statusOpt).map { needReschedule => Option(deploymentData.id).filter(_ => needReschedule) } }) @@ -259,9 +258,9 @@ class PeriodicProcessService( private def synchronizeDeploymentState( deployment: ScheduleDeploymentData, processState: Option[StatusDetails] - ): RepositoryAction[NeedsReschedule] = { - implicit class RichRepositoryAction[Unit](a: RepositoryAction[Unit]) { - def needsReschedule(value: Boolean): RepositoryAction[NeedsReschedule] = a.map(_ => value) + ): Future[NeedsReschedule] = { + implicit class RichFuture[Unit](a: Future[Unit]) { + def needsReschedule(value: Boolean): Future[NeedsReschedule] = a.map(_ => value) } processState.map(_.status) match { case Some(status) @@ -283,21 +282,21 @@ class PeriodicProcessService( // so freshly deployed deployments aren't considered markFinished(deployment, processState).needsReschedule(value = true) case _ => - scheduledProcessesRepository.monad.pure(()).needsReschedule(value = false) + Future.successful(()).needsReschedule(value = false) } } private def reschedule( processScheduleData: PeriodicProcessScheduleData, needRescheduleDeploymentIds: Set[PeriodicProcessDeploymentId] - ): RepositoryAction[Callback] = { + ): Future[Callback] = { import processScheduleData._ val scheduleActions = deployments.map { deployment => if (needRescheduleDeploymentIds.contains(deployment.id)) - deployment.nextRunAt(clock) match { + nextRunAt(deployment, clock) match { case Right(Some(futureDate)) => logger.info(s"Rescheduling ${deployment.display} to $futureDate") - val action = scheduledProcessesRepository + val action = periodicProcessesManager .schedule(process.id, deployment.scheduleName, futureDate, deploymentRetryConfig.deployMaxRetries) .flatMap { data => handleEvent(ScheduledEvent(data, firstSchedule = false)) @@ -313,7 +312,7 @@ class PeriodicProcessService( else Option(deployment) .filter(_.state.status == PeriodicProcessDeploymentStatus.Scheduled) - .map(_ => scheduledProcessesRepository.monad.pure(())) + .map(_ => Future.successful(())) } @@ -327,18 +326,26 @@ class PeriodicProcessService( scheduleActions.flatten.sequence.as(emptyCallback) } - private def markFinished(deployment: ScheduleDeploymentData, state: Option[StatusDetails]): RepositoryAction[Unit] = { + private def nextRunAt(deployment: PeriodicProcessDeployment, clock: Clock): Either[String, Option[LocalDateTime]] = + (fromApi(deployment.periodicProcess.scheduleProperty), deployment.scheduleName.value) match { + case (MultipleScheduleProperty(schedules), Some(name)) => + schedules.get(name).toRight(s"Failed to find schedule: $deployment.scheduleName").flatMap(_.nextRunAt(clock)) + case (e: SingleScheduleProperty, None) => e.nextRunAt(clock) + case (schedule, name) => Left(s"Schedule name: $name mismatch with schedule: $schedule") + } + + private def markFinished(deployment: ScheduleDeploymentData, state: Option[StatusDetails]): Future[Unit] = { logger.info(s"Marking ${deployment.display} with status: ${deployment.state.status} as finished") for { - _ <- scheduledProcessesRepository.markFinished(deployment.id) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFinished(deployment.id) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FinishedEvent(currentState, state)) } private def handleFailedDeployment( - deployment: PeriodicProcessDeployment[_], + deployment: PeriodicProcessDeployment, state: Option[StatusDetails] - ): RepositoryAction[Unit] = { + ): Future[Unit] = { def calculateNextRetryAt = now().plus(deploymentRetryConfig.deployRetryPenalize.toMillis, ChronoUnit.MILLIS) val retriesLeft = @@ -357,19 +364,19 @@ class PeriodicProcessService( ) for { - _ <- scheduledProcessesRepository.markFailedOnDeployWithStatus(deployment.id, status, retriesLeft, nextRetryAt) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFailedOnDeployWithStatus(deployment.id, status, retriesLeft, nextRetryAt) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FailedOnDeployEvent(currentState, state)) } private def markFailedAction( deployment: ScheduleDeploymentData, state: Option[StatusDetails] - ): RepositoryAction[Unit] = { + ): Future[Unit] = { logger.info(s"Marking ${deployment.display} as failed.") for { - _ <- scheduledProcessesRepository.markFailed(deployment.id) - currentState <- scheduledProcessesRepository.findProcessData(deployment.id) + _ <- periodicProcessesManager.markFailed(deployment.id) + currentState <- periodicProcessesManager.findProcessData(deployment.id) } yield handleEvent(FailedOnRunEvent(currentState, state)) } @@ -380,26 +387,26 @@ class PeriodicProcessService( _ <- activeSchedules.groupedByPeriodicProcess.map(p => deactivateAction(p.process)).sequence.runWithCallbacks } yield runningDeploymentsForSchedules.map(deployment => DeploymentId(deployment.toString)) - private def deactivateAction(process: PeriodicProcess[_]): RepositoryAction[Callback] = { + private def deactivateAction(process: PeriodicProcess): Future[Callback] = { logger.info(s"Deactivate periodic process id: ${process.id.value}") for { - _ <- scheduledProcessesRepository.markInactive(process.id) + _ <- periodicProcessesManager.markInactive(process.id) // we want to delete jars only after we successfully mark process as inactive. It's better to leave jar garbage than // have process without jar - } yield () => jarManager.deleteJar(process.deploymentData.jarFileName) + } yield () => periodicDeploymentHandler.cleanAfterDeployment(process.deploymentData.runtimeParams) } private def markProcessActionExecutionFinished( processActionIdOption: Option[ProcessActionId] - ): RepositoryAction[Callback] = - scheduledProcessesRepository.monad.pure { () => + ): Future[Callback] = + Future.successful { () => processActionIdOption .map(actionService.markActionExecutionFinished) .sequence .map(_ => ()) } - def deploy(deployment: PeriodicProcessDeployment[CanonicalProcess]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { // TODO: set status before deployment? val id = deployment.id val deploymentData = DeploymentData( @@ -417,38 +424,48 @@ class PeriodicProcessService( _ <- Future.successful( logger.info("Deploying scenario {} for deployment id {}", deploymentWithJarData.processVersion, id) ) + processName = deploymentWithJarData.processVersion.processName + versionId = deploymentWithJarData.processVersion.versionId + canonicalProcessOrError <- periodicProcessesManager.fetchCanonicalProcess(processName, versionId) + canonicalProcess = canonicalProcessOrError.getOrElse { + throw new PeriodicProcessException( + s"Could not fetch CanonicalProcess for processName=$processName, versionId=$versionId" + ) + } enrichedProcessConfig <- processConfigEnricher.onDeploy( ProcessConfigEnricher.DeployData( - deploymentWithJarData.process, deploymentWithJarData.inputConfigDuringExecutionJson, - deployment + deployment, ) ) enrichedDeploymentWithJarData = deploymentWithJarData.copy(inputConfigDuringExecutionJson = enrichedProcessConfig.inputConfigDuringExecutionJson ) - externalDeploymentId <- jarManager.deployWithJar(enrichedDeploymentWithJarData, deploymentData) + externalDeploymentId <- periodicDeploymentHandler.deployWithRuntimeParams( + enrichedDeploymentWithJarData, + deploymentData, + canonicalProcess, + ) } yield externalDeploymentId deploymentAction .flatMap { externalDeploymentId => logger.info("Scenario has been deployed {} for deployment id {}", deploymentWithJarData.processVersion, id) // TODO: add externalDeploymentId?? - scheduledProcessesRepository + periodicProcessesManager .markDeployed(id) - .flatMap(_ => scheduledProcessesRepository.findProcessData(id)) + .flatMap(_ => periodicProcessesManager.findProcessData(id)) .flatMap(afterChange => handleEvent(DeployedEvent(afterChange, externalDeploymentId))) - .run } // We can recover since deployment actor watches only future completion. .recoverWith { case exception => logger.error(s"Scenario deployment ${deployment.display} failed", exception) - handleFailedDeployment(deployment, None).run + handleFailedDeployment(deployment, None) } } // TODO: allow access to DB in listener? - private def handleEvent(event: PeriodicProcessEvent): scheduledProcessesRepository.Action[Unit] = { - scheduledProcessesRepository.monad.pure { + private def handleEvent(event: PeriodicProcessEvent): Future[Unit] = { + Future.successful { try { periodicProcessListener.onPeriodicProcessEvent.applyOrElse(event, (_: PeriodicProcessEvent) => ()) } catch { @@ -507,20 +524,22 @@ class PeriodicProcessService( processName: ProcessName, deploymentsPerScheduleMaxCount: Int = 1 ): Future[SchedulesState] = - scheduledProcessesRepository.getLatestDeploymentsForActiveSchedules(processName, deploymentsPerScheduleMaxCount).run + periodicProcessesManager.getLatestDeploymentsForActiveSchedules( + processName, + deploymentsPerScheduleMaxCount, + ) def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, deploymentsPerScheduleMaxCount: Int ): Future[SchedulesState] = - scheduledProcessesRepository + periodicProcessesManager .getLatestDeploymentsForLatestInactiveSchedules( processName, inactiveProcessesMaxCount, - deploymentsPerScheduleMaxCount + deploymentsPerScheduleMaxCount, ) - .run implicit class RuntimeStatusesExt(runtimeStatuses: List[StatusDetails]) { @@ -531,7 +550,7 @@ class PeriodicProcessService( } private def scheduledExecutionStatusAndDateFinished( - entity: PeriodicProcessDeployment[Unit], + entity: PeriodicProcessDeployment, ): Option[FinishedScheduledExecutionMetadata] = { for { status <- entity.state.status match { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala similarity index 90% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala index 30d913f0d3a..70f26169c6f 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManager.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicProcessStateDefinitionManager.scala @@ -1,11 +1,11 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import pl.touk.nussknacker.engine.api.deployment.{ OverridingProcessStateDefinitionManager, ProcessStateDefinitionManager, StateStatus } -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} class PeriodicProcessStateDefinitionManager(delegate: ProcessStateDefinitionManager) extends OverridingProcessStateDefinitionManager( diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala similarity index 97% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala index ba14646c577..ba42c89deb0 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicStateStatus.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/PeriodicStateStatus.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import pl.touk.nussknacker.engine.api.deployment.StateStatus.StatusName import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala similarity index 84% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala index d53b95c95c0..f4ed347af3e 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/RescheduleFinishedActor.scala @@ -1,8 +1,8 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{Actor, Props, Timers} import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.management.periodic.RescheduleFinishedActor.{CheckStates, CheckStatesCompleted} +import pl.touk.nussknacker.engine.common.periodic.RescheduleFinishedActor.{CheckStates, CheckStatesCompleted} import scala.concurrent.Future import scala.concurrent.duration._ @@ -14,7 +14,7 @@ object RescheduleFinishedActor { props(service.handleFinished, interval) } - private[periodic] def props(handleFinished: => Future[Unit], interval: FiniteDuration): Props = { + private[engine] def props(handleFinished: => Future[Unit], interval: FiniteDuration): Props = { Props(new RescheduleFinishedActor(handleFinished, interval)) } diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala similarity index 95% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala index 989d625cb51..d692045356c 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractor.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractor.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.instances.list._ import cats.syntax.traverse._ import com.typesafe.scalalogging.LazyLogging import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName +import pl.touk.nussknacker.engine.common.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName import java.time.Clock diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala similarity index 70% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala index 209f9adb675..6acc84c23f4 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SchedulePropertyExtractorFactory.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SchedulePropertyExtractorFactory.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.typesafe.config.Config diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala similarity index 64% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala index 97b7aa80af9..91e23eeb792 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/SingleScheduleProperty.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/SingleScheduleProperty.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.model.time.ExecutionTime @@ -6,12 +6,44 @@ import com.cronutils.model.{Cron, CronType} import com.cronutils.parser.CronParser import io.circe.generic.JsonCodec import io.circe.generic.extras.{Configuration, ConfiguredJsonCodec} +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} +import scala.collection.compat._ import scala.util.Try object ScheduleProperty { implicit val configuration: Configuration = Configuration.default.withDefaults.withDiscriminator("type") + + def toApi(scheduleProperty: ScheduleProperty): PeriodicProcessesManager.ScheduleProperty = scheduleProperty match { + case MultipleScheduleProperty(schedules) => + PeriodicProcessesManager.MultipleScheduleProperty(schedules.view.mapValues(singleToApi).toMap) + case p: CronScheduleProperty => + singleToApi(p) + } + + private def singleToApi( + scheduleProperty: SingleScheduleProperty + ): PeriodicProcessesManager.SingleScheduleProperty = scheduleProperty match { + case CronScheduleProperty(labelOrCronExpr) => + PeriodicProcessesManager.CronScheduleProperty(labelOrCronExpr) + } + + def fromApi( + scheduleProperty: PeriodicProcessesManager.ScheduleProperty + ): ScheduleProperty = scheduleProperty match { + case PeriodicProcessesManager.MultipleScheduleProperty(schedules) => + MultipleScheduleProperty(schedules.view.mapValues(singleFromApi).toMap) + case p: PeriodicProcessesManager.CronScheduleProperty => + singleFromApi(p) + } + + private def singleFromApi( + singleScheduleProperty: PeriodicProcessesManager.SingleScheduleProperty + ): SingleScheduleProperty = singleScheduleProperty match { + case PeriodicProcessesManager.CronScheduleProperty(labelOrCronExpr) => CronScheduleProperty(labelOrCronExpr) + } + } @ConfiguredJsonCodec sealed trait ScheduleProperty @@ -34,7 +66,7 @@ object SingleScheduleProperty { @JsonCodec case class CronScheduleProperty(labelOrCronExpr: String) extends SingleScheduleProperty { import cats.implicits._ - import pl.touk.nussknacker.engine.management.periodic.CronScheduleProperty._ + import pl.touk.nussknacker.engine.common.periodic.CronScheduleProperty._ private lazy val cronsOrError: Either[String, List[Cron]] = { val (errors, crons) = labelOrCronExpr diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala similarity index 95% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala index b3fe622b411..35bd201f79b 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/Utils.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/Utils.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import akka.actor.{ActorRef, ActorSystem, Props} import com.typesafe.scalalogging.LazyLogging diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala similarity index 83% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala index 9501316c622..74f08627555 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/WithRunNowPeriodicCustomActionsProviderFactory.scala @@ -1,12 +1,11 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.common.periodic import cats.data.OptionT import com.typesafe.scalalogging.LazyLogging -import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.{DMCustomActionCommand, ScenarioActionName} import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.deployment.{CustomActionDefinition, CustomActionResult} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository -import pl.touk.nussknacker.engine.api.deployment.DMCustomActionCommand import java.net.URI import scala.concurrent.{ExecutionContext, Future} @@ -14,11 +13,11 @@ import scala.concurrent.{ExecutionContext, Future} class WithRunNowPeriodicCustomActionsProviderFactory extends PeriodicCustomActionsProviderFactory { override def create( - periodicProcessesRepository: PeriodicProcessesRepository, - service: PeriodicProcessService + periodicProcessesManager: PeriodicProcessesManager, + service: PeriodicProcessService, + processingType: String, ): PeriodicCustomActionsProvider = new PeriodicCustomActionsProvider with LazyLogging { implicit val ec: ExecutionContext = ExecutionContext.global - import periodicProcessesRepository._ override def customActions: List[CustomActionDefinition] = List(InstantBatchCustomAction()) @@ -46,7 +45,7 @@ class WithRunNowPeriodicCustomActionsProviderFactory extends PeriodicCustomActio .map(_.groupedByPeriodicProcess.headOption.flatMap(_.deployments.headOption)) ) processDeploymentWithProcessJson <- OptionT.liftF( - periodicProcessesRepository.findProcessData(processDeployment.id).run + periodicProcessesManager.findProcessData(processDeployment.id) ) _ <- OptionT.liftF(service.deploy(processDeploymentWithProcessJson)) } yield () diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala similarity index 92% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala index 9eddcdbde26..c701c175cdc 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/cron/CronParameterValidator.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/cron/CronParameterValidator.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic.cron +package pl.touk.nussknacker.engine.common.periodic.cron import cats.data.Validated import cats.data.Validated.{invalid, valid} @@ -12,7 +12,7 @@ import pl.touk.nussknacker.engine.api.definition.{ } import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.graph.expression.Expression -import pl.touk.nussknacker.engine.management.periodic.SchedulePropertyExtractor +import pl.touk.nussknacker.engine.common.periodic.SchedulePropertyExtractor object CronParameterValidator extends CronParameterValidator { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala similarity index 71% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala index 1871cf05518..2fcdfcaab12 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/AdditionalDeploymentDataProvider.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/AdditionalDeploymentDataProvider.scala @@ -1,19 +1,19 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment import java.time.format.DateTimeFormatter trait AdditionalDeploymentDataProvider { - def prepareAdditionalData(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Map[String, String] + def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] } object DefaultAdditionalDeploymentDataProvider extends AdditionalDeploymentDataProvider { - override def prepareAdditionalData(runDetails: PeriodicProcessDeployment[CanonicalProcess]): Map[String, String] = { + override def prepareAdditionalData(runDetails: PeriodicProcessDeployment): Map[String, String] = { Map( "deploymentId" -> runDetails.id.value.toString, "runAt" -> runDetails.runAt.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME), diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala similarity index 68% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala index 73a092e9c57..80ef76f6564 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/PeriodicProcessListener.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/PeriodicProcessListener.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.Config import pl.touk.nussknacker.engine.api.deployment.StatusDetails +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.ExternalDeploymentId -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment /* Listener is at-least-once. If there are problems e.g. with DB, invocation can be repeated for same event. @@ -21,29 +21,28 @@ trait PeriodicProcessListenerFactory { } sealed trait PeriodicProcessEvent { - val deployment: PeriodicProcessDeployment[CanonicalProcess] + val deployment: PeriodicProcessDeployment } case class DeployedEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, externalDeploymentId: Option[ExternalDeploymentId] ) extends PeriodicProcessEvent -case class FinishedEvent(deployment: PeriodicProcessDeployment[CanonicalProcess], processState: Option[StatusDetails]) +case class FinishedEvent(deployment: PeriodicProcessDeployment, processState: Option[StatusDetails]) extends PeriodicProcessEvent case class FailedOnDeployEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent case class FailedOnRunEvent( - deployment: PeriodicProcessDeployment[CanonicalProcess], + deployment: PeriodicProcessDeployment, processState: Option[StatusDetails] ) extends PeriodicProcessEvent -case class ScheduledEvent(deployment: PeriodicProcessDeployment[CanonicalProcess], firstSchedule: Boolean) - extends PeriodicProcessEvent +case class ScheduledEvent(deployment: PeriodicProcessDeployment, firstSchedule: Boolean) extends PeriodicProcessEvent object EmptyListener extends EmptyListener diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala similarity index 83% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala rename to engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala index 1cdf3177953..df48d370dae 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/service/ProcessConfigEnricher.scala +++ b/engine/common/periodic-deployment-manager/src/main/scala/pl/touk/nussknacker/engine/common/periodic/service/ProcessConfigEnricher.scala @@ -1,9 +1,9 @@ -package pl.touk.nussknacker.engine.management.periodic.service +package pl.touk.nussknacker.engine.common.periodic.service import com.typesafe.config.{Config, ConfigFactory} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.service.ProcessConfigEnricher.{ +import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.{ DeployData, EnrichedProcessConfig, InitialScheduleData @@ -31,7 +31,6 @@ trait ProcessConfigEnricher { object ProcessConfigEnricher { trait ProcessConfigEnricherInputData { - def canonicalProcess: CanonicalProcess def inputConfigDuringExecutionJson: String def inputConfigDuringExecution: Config = { @@ -40,13 +39,11 @@ object ProcessConfigEnricher { } - case class InitialScheduleData(canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String) - extends ProcessConfigEnricherInputData + case class InitialScheduleData(inputConfigDuringExecutionJson: String) extends ProcessConfigEnricherInputData case class DeployData( - canonicalProcess: CanonicalProcess, inputConfigDuringExecutionJson: String, - deployment: PeriodicProcessDeployment[CanonicalProcess] + deployment: PeriodicProcessDeployment ) extends ProcessConfigEnricherInputData case class EnrichedProcessConfig(inputConfigDuringExecutionJson: String) diff --git a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider index e3b1553f5e4..76e20a0b331 100644 --- a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider +++ b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.DeploymentManagerProvider @@ -1 +1 @@ -pl.touk.nussknacker.engine.management.periodic.FlinkPeriodicDeploymentManagerProvider +pl.touk.nussknacker.engine.management.periodic.flink.FlinkPeriodicDeploymentManagerProvider diff --git a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator index 202f7c5d586..82cb6147dba 100644 --- a/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator +++ b/engine/flink/management/periodic/src/main/resources/META-INF/services/pl.touk.nussknacker.engine.api.definition.CustomParameterValidator @@ -1 +1 @@ -pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator \ No newline at end of file +pl.touk.nussknacker.engine.management.periodic.flink.cron.CronParameterValidator diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala deleted file mode 100644 index 255ab8108e9..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/JarManager.scala +++ /dev/null @@ -1,23 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic - -import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData - -import scala.concurrent.Future - -private[periodic] trait JarManager { - - def prepareDeploymentWithJar( - processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData[CanonicalProcess]] - - def deployWithJar( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - deploymentData: DeploymentData, - ): Future[Option[ExternalDeploymentId]] - - def deleteJar(jarFileName: String): Future[Unit] -} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala deleted file mode 100644 index bb65cef5794..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/DbInitializer.scala +++ /dev/null @@ -1,56 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.db - -import com.github.tminglei.slickpg.ExPostgresProfile -import com.typesafe.config.Config -import com.typesafe.scalalogging.LazyLogging -import net.ceedubs.ficus.readers.ValueReader -import org.flywaydb.core.Flyway -import org.flywaydb.core.api.configuration.FluentConfiguration -import org.flywaydb.core.internal.database.postgresql.PostgreSQLDatabaseType -import slick.jdbc.{HsqldbProfile, JdbcBackend, JdbcProfile, PostgresProfile} - -object DbInitializer extends LazyLogging { - - def init(configDb: Config): (JdbcBackend.DatabaseDef, JdbcProfile) = { - import net.ceedubs.ficus.Ficus._ - val url = configDb.as[String]("url") - val profile = chooseDbProfile(url) - logger.info("Applying db migrations") - - // we want to set property on FluentConfiguration only if there is property in config - implicit class OptionalConfig(config: FluentConfiguration) { - def setOptional[A](name: String, setAction: (FluentConfiguration, A) => FluentConfiguration)( - implicit reader: ValueReader[Option[A]] - ): FluentConfiguration = { - configDb.getAs[A](name).fold(config)(setAction(config, _)) - } - } - - Flyway - .configure() - .locations( - (profile match { - case _: HsqldbProfile => Array("db/batch_periodic/migration/hsql", "db/batch_periodic/migration/common") - case _: PostgresProfile => Array("db/batch_periodic/migration/postgres", "db/batch_periodic/migration/common") - case _ => - throw new IllegalArgumentException(s"Unsupported database url: $url. Use either PostgreSQL or HSQLDB.") - }): _* - ) - .dataSource(url, configDb.as[String]("user"), configDb.as[String]("password")) - .setOptional[String]("schema", _.schemas(_)) - .setOptional[String]("table", _.table(_)) - .baselineOnMigrate(true) - .load() - .migrate() - - (JdbcBackend.Database.forConfig(path = "", configDb), profile) - } - - private def chooseDbProfile(dbUrl: String): JdbcProfile = { - dbUrl match { - case url if (new PostgreSQLDatabaseType).handlesJDBCUrl(url) => ExPostgresProfile - case _ => HsqldbProfile - } - } - -} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala deleted file mode 100644 index 229432ed223..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/db/PeriodicProcessesTable.scala +++ /dev/null @@ -1,233 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.db - -import io.circe.syntax._ -import pl.touk.nussknacker.engine.api.deployment.ProcessActionId -import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessId -import pl.touk.nussknacker.engine.marshall.ProcessMarshaller -import slick.ast.TypedType -import slick.jdbc.JdbcProfile -import slick.lifted.MappedToBase.mappedToIsomorphism -import slick.lifted.ProvenShape -import slick.sql.SqlProfile.ColumnOption.NotNull - -import java.time.LocalDateTime -import java.util.UUID - -trait PeriodicProcessesTableFactory { - - protected val profile: JdbcProfile - - import profile.api._ - - implicit val processNameMapping: BaseColumnType[ProcessName] = - MappedColumnType.base[ProcessName, String](_.value, ProcessName.apply) - - implicit val versionIdMapping: BaseColumnType[VersionId] = - MappedColumnType.base[VersionId, Long](_.value, VersionId(_)) - - implicit val ProcessActionIdTypedType: TypedType[ProcessActionId] = - MappedColumnType.base[ProcessActionId, UUID]( - _.value, - ProcessActionId(_) - ) - - abstract class PeriodicProcessesTable(tag: Tag) extends Table[PeriodicProcessEntity](tag, "periodic_processes") { - - def id: Rep[PeriodicProcessId] = column[PeriodicProcessId]("id", O.PrimaryKey, O.AutoInc) - - def processName: Rep[ProcessName] = column[ProcessName]("process_name", NotNull) - - def processVersionId: Rep[VersionId] = column[VersionId]("process_version_id", NotNull) - - def processingType: Rep[String] = column[String]("processing_type", NotNull) - - def inputConfigDuringExecutionJson: Rep[String] = column[String]("input_config_during_execution", NotNull) - - def jarFileName: Rep[String] = column[String]("jar_file_name", NotNull) - - def scheduleProperty: Rep[String] = column[String]("schedule_property", NotNull) - - def active: Rep[Boolean] = column[Boolean]("active", NotNull) - - def createdAt: Rep[LocalDateTime] = column[LocalDateTime]("created_at", NotNull) - - def processActionId: Rep[Option[ProcessActionId]] = column[Option[ProcessActionId]]("process_action_id") - - } - - class PeriodicProcessesWithJsonTable(tag: Tag) extends PeriodicProcessesTable(tag) { - - def processJson: Rep[String] = column[String]("process_json", NotNull) - - override def * : ProvenShape[PeriodicProcessEntity] = ( - id, - processName, - processVersionId, - processingType, - processJson, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) <> ( - (PeriodicProcessEntity.createWithJson _).tupled, - (e: PeriodicProcessEntity) => - PeriodicProcessEntity.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - processJson, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - processJson.asJson.noSpaces, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) - - } - - class PeriodicProcessWithoutJson(tag: Tag) extends PeriodicProcessesTable(tag) { - - override def * : ProvenShape[PeriodicProcessEntity] = ( - id, - processName, - processVersionId, - processingType, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) <> ( - (PeriodicProcessEntity.createWithoutJson _).tupled, - (e: PeriodicProcessEntity) => - PeriodicProcessEntity.unapply(e).map { - case ( - id, - processName, - versionId, - processingType, - _, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) => - ( - id, - processName, - versionId, - processingType, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - } - ) - - } - - object PeriodicProcessesWithJson extends TableQuery(new PeriodicProcessesWithJsonTable(_)) - - object PeriodicProcessesWithoutJson extends TableQuery(new PeriodicProcessWithoutJson(_)) - -} - -object PeriodicProcessEntity { - - def createWithJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - processJson: String, - inputConfigDuringExecutionJson: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntity = - PeriodicProcessEntity( - id, - processName, - processVersionId, - processingType, - Some(ProcessMarshaller.fromJsonUnsafe(processJson)), - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - - def createWithoutJson( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - inputConfigDuringExecutionJson: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] - ): PeriodicProcessEntity = - PeriodicProcessEntity( - id, - processName, - processVersionId, - processingType, - None, - inputConfigDuringExecutionJson, - jarFileName, - scheduleProperty, - active, - createdAt, - processActionId - ) - -} - -case class PeriodicProcessEntity( - id: PeriodicProcessId, - processName: ProcessName, - processVersionId: VersionId, - processingType: String, - processJson: Option[CanonicalProcess], - inputConfigDuringExecutionJson: String, - jarFileName: String, - scheduleProperty: String, - active: Boolean, - createdAt: LocalDateTime, - processActionId: Option[ProcessActionId] -) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala similarity index 50% rename from engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala rename to engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala index 297d21a1d7c..e7fcdc26562 100644 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkJarManager.scala +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentHandler.scala @@ -2,12 +2,12 @@ package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.scalalogging.LazyLogging import org.apache.flink.api.common.JobID -import pl.touk.nussknacker.engine.{BaseModelData, newdeployment} import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData -import pl.touk.nussknacker.engine.management.periodic.{JarManager, PeriodicBatchConfig} +import pl.touk.nussknacker.engine.management.periodic.flink.FlinkPeriodicDeploymentHandler.jarFileNameRuntimeParam import pl.touk.nussknacker.engine.management.rest.{FlinkClient, HttpFlinkClient} import pl.touk.nussknacker.engine.management.{ FlinkConfig, @@ -16,20 +16,23 @@ import pl.touk.nussknacker.engine.management.{ FlinkStreamingRestManager } import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution +import pl.touk.nussknacker.engine.{BaseModelData, newdeployment} import sttp.client3.SttpBackend import java.nio.file.{Files, Path, Paths} import scala.concurrent.{ExecutionContext, Future} -private[periodic] object FlinkJarManager { +private[periodic] object FlinkPeriodicDeploymentHandler { + + val jarFileNameRuntimeParam = "jarFileName" - def apply(flinkConfig: FlinkConfig, periodicBatchConfig: PeriodicBatchConfig, modelData: BaseModelData)( + def apply(flinkConfig: FlinkConfig, jarsDir: String, modelData: BaseModelData)( implicit backend: SttpBackend[Future, Any], ec: ExecutionContext - ): JarManager = { - new FlinkJarManager( + ): PeriodicDeploymentHandler = { + new FlinkPeriodicDeploymentHandler( flinkClient = HttpFlinkClient.createUnsafe(flinkConfig), - jarsDir = Paths.get(periodicBatchConfig.jarsDir), + jarsDir = Paths.get(jarsDir), inputConfigDuringExecution = modelData.inputConfigDuringExecution, modelJarProvider = new FlinkModelJarProvider(modelData.modelClassLoaderUrls) ) @@ -38,27 +41,25 @@ private[periodic] object FlinkJarManager { } // Used by [[PeriodicProcessService]]. -private[periodic] class FlinkJarManager( +class FlinkPeriodicDeploymentHandler( flinkClient: FlinkClient, jarsDir: Path, inputConfigDuringExecution: InputConfigDuringExecution, modelJarProvider: FlinkModelJarProvider -) extends JarManager +) extends PeriodicDeploymentHandler with LazyLogging { import scala.concurrent.ExecutionContext.Implicits.global - override def prepareDeploymentWithJar( + override def prepareDeploymentWithRuntimeParams( processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData[CanonicalProcess]] = { + ): Future[DeploymentWithRuntimeParams] = { logger.info(s"Prepare deployment for scenario: $processVersion") copyJarToLocalDir(processVersion).map { jarFileName => - DeploymentWithJarData( + DeploymentWithRuntimeParams( processVersion = processVersion, - process = canonicalProcess, inputConfigDuringExecutionJson = inputConfigDuringExecution.serialized, - jarFileName = jarFileName + runtimeParams = RuntimeParams(Map(jarFileNameRuntimeParam -> jarFileName)) ) } } @@ -73,36 +74,52 @@ private[periodic] class FlinkJarManager( jarFileName } - override def deployWithJar( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - deploymentData: DeploymentData + override def deployWithRuntimeParams( + deployment: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, ): Future[Option[ExternalDeploymentId]] = { - val processVersion = deploymentWithJarData.processVersion - logger.info( - s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: ${deploymentWithJarData.jarFileName}" - ) - val jarFile = jarsDir.resolve(deploymentWithJarData.jarFileName).toFile - val args = FlinkDeploymentManager.prepareProgramArgs( - deploymentWithJarData.inputConfigDuringExecutionJson, - processVersion, - deploymentData, - deploymentWithJarData.process - ) - flinkClient.runProgram( - jarFile, - FlinkStreamingRestManager.MainClassName, - args, - None, - deploymentData.deploymentId.toNewDeploymentIdOpt.map(toJobId) - ) + val processVersion = deployment.processVersion + deployment.runtimeParams.params.get(jarFileNameRuntimeParam) match { + case Some(jarFileName) => + logger.info( + s"Deploying scenario ${processVersion.processName}, version id: ${processVersion.versionId} and jar: $jarFileName" + ) + val jarFile = jarsDir.resolve(jarFileName).toFile + val args = FlinkDeploymentManager.prepareProgramArgs( + deployment.inputConfigDuringExecutionJson, + processVersion, + deploymentData, + canonicalProcess, + ) + flinkClient.runProgram( + jarFile, + FlinkStreamingRestManager.MainClassName, + args, + None, + deploymentData.deploymentId.toNewDeploymentIdOpt.map(toJobId) + ) + case None => + logger.error( + s"Cannot deploy scenario ${processVersion.processName}, version id: ${processVersion.versionId}: jar file name not present" + ) + Future.successful(None) + } } - override def deleteJar(jarFileName: String): Future[Unit] = { - logger.info(s"Deleting jar: $jarFileName") - for { - _ <- deleteLocalJar(jarFileName) - _ <- flinkClient.deleteJarIfExists(jarFileName) - } yield () + override def cleanAfterDeployment(runtimeParams: RuntimeParams): Future[Unit] = { + runtimeParams.params.get(jarFileNameRuntimeParam) match { + case Some(jarFileName) => + logger.info(s"Deleting jar: $jarFileName") + for { + _ <- deleteLocalJar(jarFileName) + _ <- flinkClient.deleteJarIfExists(jarFileName) + } yield () + case None => + logger.warn(s"Jar file name not present among runtime params: ${runtimeParams}") + Future.unit + } + } private def deleteLocalJar(jarFileName: String): Future[Unit] = Future { diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala new file mode 100644 index 00000000000..ce889eb3a23 --- /dev/null +++ b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkPeriodicDeploymentManagerProvider.scala @@ -0,0 +1,37 @@ +package pl.touk.nussknacker.engine.management.periodic.flink + +import com.typesafe.config.Config +import pl.touk.nussknacker.engine.common.periodic.{ + PeriodicBatchConfig, + PeriodicDeploymentHandler, + PeriodicDeploymentManagerProvider +} +import pl.touk.nussknacker.engine.management.{FlinkConfig, FlinkStreamingDeploymentManagerProvider} +import pl.touk.nussknacker.engine.util.config.ConfigEnrichments.RichConfig +import pl.touk.nussknacker.engine.{BaseModelData, DeploymentManagerDependencies} + +class FlinkPeriodicDeploymentManagerProvider + extends PeriodicDeploymentManagerProvider( + name = "flinkPeriodic", + delegate = new FlinkStreamingDeploymentManagerProvider(), + ) { + + override protected def createPeriodicDeploymentHandler( + modelData: BaseModelData, + dependencies: DeploymentManagerDependencies, + config: Config + ): PeriodicDeploymentHandler = { + import dependencies._ + import net.ceedubs.ficus.Ficus._ + import net.ceedubs.ficus.readers.ArbitraryTypeReader._ + val periodicBatchConfig = config.as[PeriodicBatchConfig]("deploymentManager") + val flinkConfig = config.rootAs[FlinkConfig] + + FlinkPeriodicDeploymentHandler( + flinkConfig = flinkConfig, + jarsDir = periodicBatchConfig.jarsDir, + modelData = modelData, + ) + } + +} diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala deleted file mode 100644 index be290b90e9a..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/DeploymentWithJarData.scala +++ /dev/null @@ -1,10 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.model - -import pl.touk.nussknacker.engine.api.ProcessVersion - -case class DeploymentWithJarData[ProcessRep]( - processVersion: ProcessVersion, - process: ProcessRep, - inputConfigDuringExecutionJson: String, - jarFileName: String -) diff --git a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala b/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala deleted file mode 100644 index cca749d2036..00000000000 --- a/engine/flink/management/periodic/src/main/scala/pl/touk/nussknacker/engine/management/periodic/model/PeriodicProcessDeployment.scala +++ /dev/null @@ -1,52 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic.model - -import pl.touk.nussknacker.engine.management.periodic.{MultipleScheduleProperty, SingleScheduleProperty} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import slick.lifted.MappedTo - -import java.time.{Clock, LocalDateTime} - -// TODO: We should separate schedules concept from deployments - fully switch to ScheduleData and ScheduleDeploymentData -case class PeriodicProcessDeployment[ProcessRep]( - id: PeriodicProcessDeploymentId, - periodicProcess: PeriodicProcess[ProcessRep], - createdAt: LocalDateTime, - runAt: LocalDateTime, - scheduleName: ScheduleName, - retriesLeft: Int, - nextRetryAt: Option[LocalDateTime], - state: PeriodicProcessDeploymentState -) { - - def nextRunAt(clock: Clock): Either[String, Option[LocalDateTime]] = - (periodicProcess.scheduleProperty, scheduleName.value) match { - case (MultipleScheduleProperty(schedules), Some(name)) => - schedules.get(name).toRight(s"Failed to find schedule: $scheduleName").flatMap(_.nextRunAt(clock)) - case (e: SingleScheduleProperty, None) => e.nextRunAt(clock) - case (schedule, name) => Left(s"Schedule name: $name mismatch with schedule: $schedule") - } - - def display: String = - s"${periodicProcess.processVersion} with scheduleName=${scheduleName.display} and deploymentId=$id" - -} - -case class PeriodicProcessDeploymentState( - deployedAt: Option[LocalDateTime], - completedAt: Option[LocalDateTime], - status: PeriodicProcessDeploymentStatus -) - -case class PeriodicProcessDeploymentId(value: Long) extends AnyVal with MappedTo[Long] { - override def toString: String = value.toString -} - -object PeriodicProcessDeploymentStatus extends Enumeration { - type PeriodicProcessDeploymentStatus = Value - - val Scheduled, Deployed, Finished, Failed, RetryingDeploy, FailedOnDeploy = Value -} - -case class ScheduleName(value: Option[String]) { - def display: String = value.getOrElse("[default]") -} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala deleted file mode 100644 index 63d37875d1c..00000000000 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerStub.scala +++ /dev/null @@ -1,38 +0,0 @@ -package pl.touk.nussknacker.engine.management.periodic - -import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData - -import scala.concurrent.Future - -class JarManagerStub extends JarManager { - - var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) - var lastDeploymentWithJarData: Option[DeploymentWithJarData[CanonicalProcess]] = None - - override def prepareDeploymentWithJar( - processVersion: ProcessVersion, - canonicalProcess: CanonicalProcess - ): Future[DeploymentWithJarData[CanonicalProcess]] = { - Future.successful( - model.DeploymentWithJarData( - processVersion = processVersion, - process = canonicalProcess, - inputConfigDuringExecutionJson = "", - jarFileName = "" - ) - ) - } - - override def deployWithJar( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - deploymentData: DeploymentData, - ): Future[Option[ExternalDeploymentId]] = { - lastDeploymentWithJarData = Some(deploymentWithJarData) - deployWithJarFuture - } - - override def deleteJar(jarFileName: String): Future[Unit] = Future.successful(()) -} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala similarity index 88% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala index 5874bd1bb26..d4d248ac99e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyExtractorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyExtractorTest.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.Inside import org.scalatest.funsuite.AnyFunSuite @@ -6,8 +6,9 @@ import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.NodeId import pl.touk.nussknacker.engine.api.parameter.ParameterName import pl.touk.nussknacker.engine.build.ScenarioBuilder +import pl.touk.nussknacker.engine.common.periodic.cron.CronParameterValidator +import pl.touk.nussknacker.engine.common.periodic.{CronScheduleProperty, CronSchedulePropertyExtractor, MultipleScheduleProperty} import pl.touk.nussknacker.engine.graph.expression.Expression -import pl.touk.nussknacker.engine.management.periodic.cron.CronParameterValidator import pl.touk.nussknacker.test.{EitherValuesDetailedMessage, ValidatedValuesDetailedMessage} class CronSchedulePropertyExtractorTest diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala similarity index 88% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala index 5fff9dcb12d..e1c06033c7e 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/CronSchedulePropertyTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/CronSchedulePropertyTest.scala @@ -1,8 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink -import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.common.periodic.CronScheduleProperty + +import java.time.{Clock, LocalDateTime, ZoneId, ZonedDateTime} class CronSchedulePropertyTest extends AnyFunSuite with Matchers { diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala similarity index 66% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala index a32745a2b53..bda0b268a96 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentActorTest.scala @@ -1,14 +1,14 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.{ActorRef, ActorSystem} import akka.testkit.{TestKit, TestKitBase, TestProbe} -import org.scalatest.LoneElement._ import org.scalatest.BeforeAndAfterAll +import org.scalatest.LoneElement._ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.DeploymentActor.CheckToBeDeployed -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeployment +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor +import pl.touk.nussknacker.engine.common.periodic.DeploymentActor.CheckToBeDeployed import scala.concurrent.Future import scala.concurrent.duration._ @@ -33,11 +33,11 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } private def shouldFindToBeDeployedScenarios( - result: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] + result: Future[Seq[PeriodicProcessDeployment]] ): Unit = { val probe = TestProbe() var counter = 0 - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { counter += 1 probe.ref ! s"invoked $counter" result @@ -54,14 +54,14 @@ class DeploymentActorTest extends AnyFunSuite with TestKitBase with Matchers wit } test("should deploy found scenario") { - val probe = TestProbe() - val waitingDeployment = PeriodicProcessDeploymentGen() - var toBeDeployed: Seq[PeriodicProcessDeployment[CanonicalProcess]] = Seq(waitingDeployment) - var actor: ActorRef = null - def findToBeDeployed: Future[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + val probe = TestProbe() + val waitingDeployment = PeriodicProcessDeploymentGen() + var toBeDeployed: Seq[PeriodicProcessDeployment] = Seq(waitingDeployment) + var actor: ActorRef = null + def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { Future.successful(toBeDeployed) } - def deploy(deployment: PeriodicProcessDeployment[CanonicalProcess]): Future[Unit] = { + def deploy(deployment: PeriodicProcessDeployment): Future[Unit] = { probe.ref ! deployment // Simulate periodic check for waiting scenarios while deploying a scenario. actor ! CheckToBeDeployed diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala similarity index 91% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala index 840b82342dc..f966cb743e9 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/DeploymentManagerStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/DeploymentManagerStub.scala @@ -1,10 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentId import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessIdWithName, ProcessName} import pl.touk.nussknacker.engine.deployment.{DeploymentId, ExternalDeploymentId} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentId import pl.touk.nussknacker.engine.testing.StubbingCommands import scala.concurrent.Future diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala similarity index 96% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala index 5c386e88208..5b39c80139a 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/FlinkClientStub.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/FlinkClientStub.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.apache.flink.configuration.Configuration import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, SavepointResult, WithDataFreshnessStatus} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala new file mode 100644 index 00000000000..0c4d2fc9cbc --- /dev/null +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerStub.scala @@ -0,0 +1,38 @@ +package pl.touk.nussknacker.engine.management.periodic.flink + +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} +import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler +import pl.touk.nussknacker.engine.deployment.{DeploymentData, ExternalDeploymentId} + +import scala.concurrent.Future + +class PeriodicDeploymentHandlerStub extends PeriodicDeploymentHandler { + + var deployWithJarFuture: Future[Option[ExternalDeploymentId]] = Future.successful(None) + var lastDeploymentWithRuntimeParams: Option[DeploymentWithRuntimeParams] = None + + override def prepareDeploymentWithRuntimeParams( + processVersion: ProcessVersion, + ): Future[DeploymentWithRuntimeParams] = { + Future.successful( + DeploymentWithRuntimeParams( + processVersion = processVersion, + inputConfigDuringExecutionJson = "", + runtimeParams = RuntimeParams(Map("jarFileName" -> "")) + ) + ) + } + + override def deployWithRuntimeParams( + deploymentWithJarData: DeploymentWithRuntimeParams, + deploymentData: DeploymentData, + canonicalProcess: CanonicalProcess, + ): Future[Option[ExternalDeploymentId]] = { + lastDeploymentWithRuntimeParams = Some(deploymentWithJarData) + deployWithJarFuture + } + + override def cleanAfterDeployment(runtimeParams: RuntimeParams): Future[Unit] = Future.successful(()) +} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala similarity index 56% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala index b90dbbdaf77..4b1c9d34aa2 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/JarManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentHandlerTest.scala @@ -1,28 +1,26 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.config.ConfigFactory import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{DeploymentWithRuntimeParams, RuntimeParams} import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} -import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess +import pl.touk.nussknacker.engine.common.periodic.PeriodicDeploymentHandler import pl.touk.nussknacker.engine.management.FlinkModelJarProvider -import pl.touk.nussknacker.engine.management.periodic.flink.FlinkJarManager -import pl.touk.nussknacker.engine.management.periodic.model.DeploymentWithJarData import pl.touk.nussknacker.engine.modelconfig.InputConfigDuringExecution import pl.touk.nussknacker.test.PatientScalaFutures import java.nio.file.{Files, Path, Paths} import scala.concurrent.Future -class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with PatientScalaFutures { +class PeriodicDeploymentHandlerTest extends AnyFunSuite with Matchers with ScalaFutures with PatientScalaFutures { private val processName = "test" private val processVersionId = 5 private val processVersion = ProcessVersion.empty.copy(processName = ProcessName(processName), versionId = VersionId(processVersionId)) - private val process = CanonicalProcess(MetaData("foo", StreamMetaData()), Nil) private val jarsDir = Files.createTempDirectory("jars-dir") private val modelJarFileContent = "abc".getBytes @@ -34,14 +32,14 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa private val currentModelUrls = List(currentModelJarFile.toURI.toURL) - private val jarManager = createJarManager(jarsDir = jarsDir) + private val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir = jarsDir) - private def createJarManager( + private def createPeriodicDeploymentHandler( jarsDir: Path, modelJarProvider: FlinkModelJarProvider = new FlinkModelJarProvider(currentModelUrls) - ): JarManager = { + ): PeriodicDeploymentHandler = { - new FlinkJarManager( + new FlinkPeriodicDeploymentHandler( flinkClient = new FlinkClientStub, jarsDir = jarsDir, inputConfigDuringExecution = InputConfigDuringExecution(ConfigFactory.empty()), @@ -50,9 +48,9 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa } test("prepareDeploymentWithJar - should copy to local dir") { - val result = jarManager.prepareDeploymentWithJar(processVersion, process) + val result = periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion) - val copiedJarFileName = result.futureValue.jarFileName + val copiedJarFileName = result.futureValue.runtimeParams.params("jarFileName") copiedJarFileName should fullyMatch regex s"^$processName-$processVersionId-\\d+\\.jar$$" val copiedJarFile = jarsDir.resolve(copiedJarFileName) Files.exists(copiedJarFile) shouldBe true @@ -60,33 +58,33 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa } test("prepareDeploymentWithJar - should handle disappearing model JAR") { - val modelJarProvider = new FlinkModelJarProvider(currentModelUrls) - val jarManager = createJarManager(jarsDir, modelJarProvider) + val modelJarProvider = new FlinkModelJarProvider(currentModelUrls) + val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir, modelJarProvider) - def verifyAndDeleteJar(result: Future[DeploymentWithJarData[CanonicalProcess]]): Unit = { - val copiedJarFile = jarsDir.resolve(result.futureValue.jarFileName) + def verifyAndDeleteJar(result: Future[DeploymentWithRuntimeParams]): Unit = { + val copiedJarFile = jarsDir.resolve(result.futureValue.runtimeParams.params("jarFileName")) Files.exists(copiedJarFile) shouldBe true Files.readAllBytes(copiedJarFile) shouldBe modelJarFileContent Files.delete(copiedJarFile) } - verifyAndDeleteJar(jarManager.prepareDeploymentWithJar(processVersion, process)) + verifyAndDeleteJar(periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion)) modelJarProvider.getJobJar().delete() shouldBe true - verifyAndDeleteJar(jarManager.prepareDeploymentWithJar(processVersion, process)) + verifyAndDeleteJar(periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion)) } test("prepareDeploymentWithJar - should create jars dir if not exists") { - val tmpDir = System.getProperty("java.io.tmpdir") - val jarsDir = Paths.get(tmpDir, s"jars-dir-not-exists-${System.currentTimeMillis()}") - val jarManager = createJarManager(jarsDir = jarsDir) + val tmpDir = System.getProperty("java.io.tmpdir") + val jarsDir = Paths.get(tmpDir, s"jars-dir-not-exists-${System.currentTimeMillis()}") + val periodicDeploymentHandler = createPeriodicDeploymentHandler(jarsDir = jarsDir) Files.exists(jarsDir) shouldBe false - val result = jarManager.prepareDeploymentWithJar(processVersion, process) + val result = periodicDeploymentHandler.prepareDeploymentWithRuntimeParams(processVersion) - val copiedJarFileName = result.futureValue.jarFileName + val copiedJarFileName = result.futureValue.runtimeParams.params("jarFileName") Files.exists(jarsDir) shouldBe true Files.exists(jarsDir.resolve(copiedJarFileName)) shouldBe true } @@ -96,13 +94,14 @@ class JarManagerTest extends AnyFunSuite with Matchers with ScalaFutures with Pa val jarPath = jarsDir.resolve(jarFileName) Files.copy(currentModelJarFile.toPath, jarPath) - jarManager.deleteJar(jarFileName).futureValue + periodicDeploymentHandler.cleanAfterDeployment(RuntimeParams(Map("jarFileName" -> jarFileName))).futureValue Files.exists(jarPath) shouldBe false } test("deleteJar - should handle not existing file") { - val result = jarManager.deleteJar("unknown.jar").futureValue + val result = + periodicDeploymentHandler.cleanAfterDeployment(RuntimeParams(Map("jarFileName" -> "unknown.jar"))).futureValue result shouldBe (()) } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala similarity index 73% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala index 37f365c1d4d..0e46f4dbb52 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicDeploymentManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicDeploymentManagerTest.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.concurrent.ScalaFutures import org.scalatest.funsuite.AnyFunSuite @@ -7,20 +7,22 @@ import org.scalatest.prop.TableDrivenPropertyChecks import org.scalatest.{Inside, OptionValues} import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessIdWithName, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{DeploymentData, User} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic.PeriodicStateStatus.{ScheduledStatus, WaitingForScheduleStatus} +import pl.touk.nussknacker.engine.common.periodic.service.{ DefaultAdditionalDeploymentDataProvider, EmptyListener, ProcessConfigEnricher } +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.deployment.{DeploymentData, User} +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager import pl.touk.nussknacker.test.PatientScalaFutures import java.time.{Clock, LocalDateTime, ZoneOffset} @@ -59,15 +61,15 @@ class PeriodicDeploymentManagerTest ) class Fixture(executionConfig: PeriodicExecutionConfig = PeriodicExecutionConfig()) { - val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val manager = new InMemPeriodicProcessesManager(processingType = "testProcessingType") val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val preparedDeploymentData = DeploymentData.withDeploymentId(UUID.randomUUID().toString) val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = repository, + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = manager, periodicProcessListener = EmptyListener, additionalDeploymentDataProvider = DefaultAdditionalDeploymentDataProvider, deploymentRetryConfig = DeploymentRetryConfig(), @@ -75,7 +77,7 @@ class PeriodicDeploymentManagerTest processConfigEnricher = ProcessConfigEnricher.identity, clock = Clock.systemDefaultZone(), new ProcessingTypeActionServiceStub, - Map.empty + Map.empty, ) val periodicDeploymentManager = new PeriodicDeploymentManager( @@ -83,6 +85,7 @@ class PeriodicDeploymentManagerTest service = periodicProcessService, schedulePropertyExtractor = CronSchedulePropertyExtractor(), EmptyPeriodicCustomActionsProvider, + "testProcessingType", toClose = () => () ) @@ -111,7 +114,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should return not deployed for scenario with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") val state = f.getMergedStatusDetails.status @@ -120,7 +123,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be scheduled when scenario scheduled and no job on Flink") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe a[ScheduledStatus] @@ -134,7 +137,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be scheduled when scenario scheduled and job finished on Flink") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -144,8 +147,8 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be finished when scenario finished and job finished on Flink") { val f = new Fixture - val periodicProcessId = f.repository.addOnlyProcess(processName, CronScheduleProperty("0 0 0 1 1 ? 1970")) - val deploymentId = f.repository.addOnlyDeployment( + val periodicProcessId = f.manager.addOnlyProcess(processName, CronScheduleProperty("0 0 0 1 1 ? 1970")) + val deploymentId = f.manager.addOnlyDeployment( periodicProcessId, PeriodicProcessDeploymentStatus.Finished, LocalDateTime.ofEpochSecond(0, 0, ZoneOffset.UTC) @@ -161,7 +164,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be running when scenario deployed and job running on Flink") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -171,7 +174,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be waiting for reschedule if job finished on Flink but scenario is still deployed") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -181,7 +184,7 @@ class PeriodicDeploymentManagerTest test("getProcessState - should be failed after unsuccessful deployment") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Failed) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Failed) val statusDetails = f.getMergedStatusDetails statusDetails.status shouldBe ProblemStateStatus.Failed @@ -220,13 +223,13 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled } test("deploy - should not cancel current schedule after trying to deploy with past date") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -240,13 +243,13 @@ class PeriodicDeploymentManagerTest .failed .futureValue - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Scheduled } test("deploy - should cancel existing scenario if already scheduled") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -259,13 +262,13 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities should have size 2 - f.repository.processEntities.map(_.active) shouldBe List(false, true) + f.manager.processEntities should have size 2 + f.manager.processEntities.map(_.active) shouldBe List(false, true) } test("should get status of failed job") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails @@ -275,7 +278,7 @@ class PeriodicDeploymentManagerTest test("should redeploy failed scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) val statusDetailsBeforeRedeploy = f.getMergedStatusDetails statusDetailsBeforeRedeploy.status shouldBe ProblemStateStatus.Failed @@ -294,8 +297,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Failed, PeriodicProcessDeploymentStatus.Scheduled ) @@ -307,7 +310,7 @@ class PeriodicDeploymentManagerTest test("should redeploy scheduled scenario") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) f.periodicDeploymentManager .processCommand( @@ -320,8 +323,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Scheduled, PeriodicProcessDeploymentStatus.Scheduled ) @@ -329,7 +332,7 @@ class PeriodicDeploymentManagerTest test("should redeploy running scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Running, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails) shouldBe List( @@ -347,8 +350,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Deployed, PeriodicProcessDeploymentStatus.Scheduled ) @@ -356,7 +359,7 @@ class PeriodicDeploymentManagerTest test("should redeploy finished scenario") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(SimpleStateStatus.Finished, Some(deploymentId)) val statusDetails = f.getMergedStatusDetails f.getAllowedActions(statusDetails) shouldBe List( @@ -374,8 +377,8 @@ class PeriodicDeploymentManagerTest ) .futureValue - f.repository.processEntities.map(_.active) shouldBe List(false, true) - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.processEntities.map(_.active) shouldBe List(false, true) + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) @@ -383,55 +386,55 @@ class PeriodicDeploymentManagerTest test("should cancel failed job after RescheduleActor handles finished") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue f.getMergedStatusDetails.status shouldEqual ProblemStateStatus.Failed - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - f.repository.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe true f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldEqual SimpleStateStatus.Canceled } test("should reschedule failed job after RescheduleActor handles finished when configured") { val f = new Fixture(executionConfig = PeriodicExecutionConfig(rescheduleOnFailure = true)) - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor f.periodicProcessService.handleFinished.futureValue f.getMergedStatusDetails.status shouldBe a[ScheduledStatus] - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Failed, PeriodicProcessDeploymentStatus.Scheduled ) - f.repository.processEntities.loneElement.active shouldBe true + f.manager.processEntities.loneElement.active shouldBe true } test("should cancel failed job before RescheduleActor handles finished") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldEqual SimpleStateStatus.Canceled } test("should cancel failed scenario after disappeared from Flink console") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) // this one is cyclically called by RescheduleActor @@ -441,22 +444,22 @@ class PeriodicDeploymentManagerTest f.delegateDeploymentManagerStub.jobStatus = None f.getMergedStatusDetails.status shouldEqual ProblemStateStatus.Failed - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - f.repository.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe true f.periodicDeploymentManager.processCommand(DMCancelScenarioCommand(processName, User("test", "Tester"))).futureValue - f.repository.processEntities.loneElement.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.processEntities.loneElement.active shouldBe false + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed f.getMergedStatusDetails.status shouldBe SimpleStateStatus.Canceled } test("should take into account only latest deployments of active schedules during merged status computation") { val f = new Fixture - val processId = f.repository.addOnlyProcess(processName) + val processId = f.manager.addOnlyProcess(processName) val firstDeploymentRunAt = LocalDateTime.of(2023, 1, 1, 10, 0) - f.repository.addOnlyDeployment(processId, PeriodicProcessDeploymentStatus.Failed, firstDeploymentRunAt) - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment(processId, PeriodicProcessDeploymentStatus.Failed, firstDeploymentRunAt) + f.manager.addOnlyDeployment( processId, PeriodicProcessDeploymentStatus.Finished, firstDeploymentRunAt.plusHours(1) @@ -469,13 +472,13 @@ class PeriodicDeploymentManagerTest "should take into account only latest inactive schedule request (periodic process) during merged status computation" ) { val f = new Fixture - val firstProcessId = f.repository.addOnlyProcess(processName) - f.repository.addOnlyDeployment(firstProcessId, PeriodicProcessDeploymentStatus.Failed) - f.repository.markInactive(firstProcessId) + val firstProcessId = f.manager.addOnlyProcess(processName) + f.manager.addOnlyDeployment(firstProcessId, PeriodicProcessDeploymentStatus.Failed) + f.manager.markInactive(firstProcessId) - val secProcessId = f.repository.addOnlyProcess(processName) - f.repository.addOnlyDeployment(secProcessId, PeriodicProcessDeploymentStatus.Finished) - f.repository.markInactive(secProcessId) + val secProcessId = f.manager.addOnlyProcess(processName) + f.manager.addOnlyDeployment(secProcessId, PeriodicProcessDeploymentStatus.Finished) + f.manager.markInactive(secProcessId) f.getMergedStatusDetails.status shouldBe SimpleStateStatus.Finished } diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala similarity index 59% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala index 31b847143f6..ccbc60e06d3 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessDeploymentGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessDeploymentGen.scala @@ -1,13 +1,6 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink -import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeployment, - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentState, - PeriodicProcessDeploymentStatus, - ScheduleName -} +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import java.time.LocalDateTime @@ -15,7 +8,7 @@ object PeriodicProcessDeploymentGen { val now: LocalDateTime = LocalDateTime.now() - def apply(): PeriodicProcessDeployment[CanonicalProcess] = { + def apply(): PeriodicProcessDeployment = { PeriodicProcessDeployment( id = PeriodicProcessDeploymentId(42), periodicProcess = PeriodicProcessGen(), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala similarity index 62% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala index 1ddf34889a1..73c40fbf67d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessGen.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessGen.scala @@ -1,23 +1,23 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import pl.touk.nussknacker.engine.api.ProcessVersion +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager.CronScheduleProperty +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName -import pl.touk.nussknacker.engine.management.periodic.model.{DeploymentWithJarData, PeriodicProcess, PeriodicProcessId} +import pl.touk.nussknacker.engine.common.periodic.CronSchedulePropertyExtractor.CronPropertyDefaultName import java.time.LocalDateTime object PeriodicProcessGen { - def apply(): PeriodicProcess[CanonicalProcess] = { + def apply(): PeriodicProcess = { PeriodicProcess( id = PeriodicProcessId(42), - deploymentData = DeploymentWithJarData( + deploymentData = DeploymentWithRuntimeParams( processVersion = ProcessVersion.empty, - process = buildCanonicalProcess(), inputConfigDuringExecutionJson = "{}", - jarFileName = "jar-file-name.jar" + runtimeParams = RuntimeParams(Map("jarFileName" -> "jar-file-name.jar")) ), scheduleProperty = CronScheduleProperty("0 0 * * * ?"), active = true, diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala similarity index 72% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala index 7da7f65421a..594c7f4bfd4 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessServiceTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessServiceTest.scala @@ -1,25 +1,28 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import com.typesafe.config.{ConfigFactory, ConfigValueFactory} +import org.scalatest.OptionValues import org.scalatest.concurrent.ScalaFutures import org.scalatest.exceptions.TestFailedException -import org.scalatest.prop.TableDrivenPropertyChecks -import org.scalatest.OptionValues import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers +import org.scalatest.prop.TableDrivenPropertyChecks import pl.touk.nussknacker.engine.api.ProcessVersion -import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId, ProcessingTypeActionServiceStub} +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model.{ + PeriodicProcessDeployment, + PeriodicProcessDeploymentStatus +} import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus -import pl.touk.nussknacker.engine.api.process.ProcessName +import pl.touk.nussknacker.engine.api.deployment.{DataFreshnessPolicy, ProcessActionId, ProcessingTypeActionServiceStub} +import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.PeriodicProcessStatus -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model.{PeriodicProcessDeployment, PeriodicProcessDeploymentStatus} -import pl.touk.nussknacker.engine.management.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig -import pl.touk.nussknacker.engine.management.periodic.service.{ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.PeriodicProcessStatus +import pl.touk.nussknacker.engine.common.periodic._ +import pl.touk.nussknacker.engine.common.periodic.service.ProcessConfigEnricher.EnrichedProcessConfig +import pl.touk.nussknacker.engine.common.periodic.service.{ AdditionalDeploymentDataProvider, DeployedEvent, FailedOnDeployEvent, @@ -31,6 +34,8 @@ import pl.touk.nussknacker.engine.management.periodic.service.{ ScheduledEvent } import pl.touk.nussknacker.test.PatientScalaFutures +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager.createPeriodicProcessDeployment import java.time.temporal.ChronoField import java.time.{Clock, LocalDate, LocalDateTime} @@ -57,15 +62,19 @@ class PeriodicProcessServiceTest private val cronInFuture = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow + 1}") private val cronInPast = CronScheduleProperty(s"0 0 6 6 9 ? ${yearNow - 1}") - private val canonicalProcess = ScenarioBuilder - .streaming(processName.value) - .source("start", "source") - .emptySink("end", "KafkaSink") + private val processVersion = ProcessVersion( + versionId = VersionId(1), + processName = processName, + processId = ProcessId(1), + labels = List.empty, + user = "testUser", + modelVersion = None, + ) class Fixture { - val repository = new db.InMemPeriodicProcessesRepository(processingType = "testProcessingType") + val manager = new InMemPeriodicProcessesManager(processingType = "testProcessingType") val delegateDeploymentManagerStub = new DeploymentManagerStub - val jarManagerStub = new JarManagerStub + val periodicDeploymentHandlerStub = new PeriodicDeploymentHandlerStub val events = new ArrayBuffer[PeriodicProcessEvent]() val additionalData = Map("testMap" -> "testValue") @@ -73,8 +82,8 @@ class PeriodicProcessServiceTest val periodicProcessService = new PeriodicProcessService( delegateDeploymentManager = delegateDeploymentManagerStub, - jarManager = jarManagerStub, - scheduledProcessesRepository = repository, + periodicDeploymentHandler = periodicDeploymentHandlerStub, + periodicProcessesManager = manager, new PeriodicProcessListener { override def onPeriodicProcessEvent: PartialFunction[PeriodicProcessEvent, Unit] = { case k => @@ -85,7 +94,7 @@ class PeriodicProcessServiceTest additionalDeploymentDataProvider = new AdditionalDeploymentDataProvider { override def prepareAdditionalData( - runDetails: PeriodicProcessDeployment[CanonicalProcess] + runDetails: PeriodicProcessDeployment ): Map[String, String] = additionalData + ("runId" -> runDetails.id.value.toString) @@ -101,7 +110,7 @@ class PeriodicProcessServiceTest EnrichedProcessConfig( initialScheduleData.inputConfigDuringExecution.withValue( "processName", - ConfigValueFactory.fromAnyRef(initialScheduleData.canonicalProcess.name.value) + ConfigValueFactory.fromAnyRef(processName.value) ) ) ) @@ -121,19 +130,19 @@ class PeriodicProcessServiceTest }, Clock.systemDefaultZone(), actionService, - Map.empty + Map.empty, ) } test("findToBeDeployed - should return scheduled and to be retried scenarios") { val fWithNoRetries = new Fixture - fWithNoRetries.repository.addActiveProcess( + fWithNoRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.FailedOnDeploy, deployMaxRetries = 0 ) - val scheduledId1 = fWithNoRetries.repository.addActiveProcess( + val scheduledId1 = fWithNoRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 0 @@ -141,12 +150,12 @@ class PeriodicProcessServiceTest fWithNoRetries.periodicProcessService.findToBeDeployed.futureValue.map(_.id) shouldBe List(scheduledId1) val fWithRetries = new Fixture - val failedId2 = fWithRetries.repository.addActiveProcess( + val failedId2 = fWithRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.FailedOnDeploy, deployMaxRetries = 1 ) - val scheduledId2 = fWithRetries.repository.addActiveProcess( + val scheduledId2 = fWithRetries.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Scheduled, deployMaxRetries = 1 @@ -156,7 +165,7 @@ class PeriodicProcessServiceTest test("findToBeDeployed - should not return scenarios with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled, processingType = "other") f.periodicProcessService.findToBeDeployed.futureValue shouldBe Symbol("empty") } @@ -164,7 +173,7 @@ class PeriodicProcessServiceTest // Flink job could disappear from Flink console. test("handleFinished - should reschedule scenario if Flink job is missing") { val f = new Fixture - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, runAt = LocalDateTime.now().minusMinutes(11), @@ -173,23 +182,23 @@ class PeriodicProcessServiceTest f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 2 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 2 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) val finished :: scheduled :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List(FinishedEvent(finished, None), ScheduledEvent(scheduled, firstSchedule = false)) } // Flink job could not be available in Flink console if checked too quickly after submit. test("handleFinished - shouldn't reschedule scenario if Flink job is missing but not deployed for long enough") { val f = new Fixture - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, runAt = LocalDateTime.now().minusSeconds(11), @@ -198,15 +207,15 @@ class PeriodicProcessServiceTest f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 1 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 1 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Deployed ) val deployed :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List.empty } @@ -214,7 +223,7 @@ class PeriodicProcessServiceTest val f = new Fixture val processActionId = randomProcessActionId val deploymentId = - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) @@ -225,16 +234,16 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe Nil - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 2 - f.repository.deploymentEntities.map(_.status) shouldBe List( + f.manager.deploymentEntities should have size 2 + f.manager.deploymentEntities.map(_.status) shouldBe List( PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled ) val finished :: scheduled :: Nil = - f.repository.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList + f.manager.deploymentEntities.map(createPeriodicProcessDeployment(processEntity, _)).toList f.events.toList shouldBe List( FinishedEvent(finished, f.delegateDeploymentManagerStub.jobStatus), ScheduledEvent(scheduled, firstSchedule = false) @@ -245,7 +254,7 @@ class PeriodicProcessServiceTest val f = new Fixture val processActionId = randomProcessActionId val deploymentId = - f.repository.addActiveProcess( + f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, processActionId = Some(processActionId) @@ -261,7 +270,7 @@ class PeriodicProcessServiceTest test("handleFinished - should deactivate process if there are no future schedules") { val f = new Fixture val processActionId = randomProcessActionId - val deploymentId = f.repository.addActiveProcess( + val deploymentId = f.manager.addActiveProcess( processName, PeriodicProcessDeploymentStatus.Deployed, scheduleProperty = cronInPast, @@ -273,26 +282,29 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe List(processActionId) - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe false - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Finished + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Finished // TODO: active should be false val event = - createPeriodicProcessDeployment(processEntity.copy(active = true), f.repository.deploymentEntities.loneElement) + createPeriodicProcessDeployment( + processEntity.copy(active = true), + f.manager.deploymentEntities.loneElement, + ) f.events.loneElement shouldBe FinishedEvent(event, f.delegateDeploymentManagerStub.jobStatus) } test("handleFinished - should not deactivate process if there is future schedule") { val f = new Fixture val scheduleProperty = MultipleScheduleProperty(Map("schedule1" -> cronInPast, "schedule2" -> cronInFuture)) - val periodicProcessId = f.repository.addOnlyProcess(processName, scheduleProperty) - f.repository.addOnlyDeployment( + val periodicProcessId = f.manager.addOnlyProcess(processName, scheduleProperty) + f.manager.addOnlyDeployment( periodicProcessId, status = PeriodicProcessDeploymentStatus.Deployed, scheduleName = Some("schedule1"), deployedAt = Some(LocalDateTime.now().minusMinutes(10)) ) - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment( periodicProcessId, status = PeriodicProcessDeploymentStatus.Scheduled, scheduleName = Some("schedule2") @@ -302,22 +314,22 @@ class PeriodicProcessServiceTest f.actionService.sentActionIds shouldBe Nil - f.repository.processEntities.loneElement.active shouldBe true - f.repository.deploymentEntities.map( + f.manager.processEntities.loneElement.active shouldBe true + f.manager.deploymentEntities.map( _.status ) should contain only (PeriodicProcessDeploymentStatus.Finished, PeriodicProcessDeploymentStatus.Scheduled) } test("handleFinished - should not reschedule scenario with different processing type") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed, processingType = "other") + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed, processingType = "other") f.periodicProcessService.handleFinished.futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities should have size 1 - f.repository.deploymentEntities.map(_.status) shouldBe List(PeriodicProcessDeploymentStatus.Deployed) + f.manager.deploymentEntities should have size 1 + f.manager.deploymentEntities.map(_.status) shouldBe List(PeriodicProcessDeploymentStatus.Deployed) f.events.toList shouldBe Symbol("empty") } @@ -325,75 +337,80 @@ class PeriodicProcessServiceTest val f = new Fixture f.periodicProcessService - .schedule(CronScheduleProperty("0 0 * * * ?"), ProcessVersion.empty, canonicalProcess, randomProcessActionId) + .schedule(CronScheduleProperty("0 0 * * * ?"), processVersion, randomProcessActionId) .futureValue - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true ConfigFactory .parseString(processEntity.inputConfigDuringExecutionJson) .getString("processName") shouldBe processName.value - val deploymentEntity = f.repository.deploymentEntities.loneElement + val deploymentEntity = f.manager.deploymentEntities.loneElement deploymentEntity.status shouldBe PeriodicProcessDeploymentStatus.Scheduled f.events.toList shouldBe List( - ScheduledEvent(createPeriodicProcessDeployment(processEntity, deploymentEntity), firstSchedule = true) + ScheduledEvent( + createPeriodicProcessDeployment(processEntity, deploymentEntity), + firstSchedule = true + ) ) } test("handleFinished - should mark as failed for failed Flink job") { val f = new Fixture - val deploymentId = f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) + val deploymentId = f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Deployed) f.delegateDeploymentManagerStub.setStateStatus(ProblemStateStatus.Failed, Some(deploymentId)) f.periodicProcessService.handleFinished.futureValue // Process is still active and has to be manually canceled by user. - val processEntity = f.repository.processEntities.loneElement + val processEntity = f.manager.processEntities.loneElement processEntity.active shouldBe true - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.Failed - val expectedDetails = createPeriodicProcessDeployment(processEntity, f.repository.deploymentEntities.head) + val expectedDetails = + createPeriodicProcessDeployment(processEntity, f.manager.deploymentEntities.head) f.events.toList shouldBe List(FailedOnRunEvent(expectedDetails, f.delegateDeploymentManagerStub.jobStatus)) } test("deploy - should deploy and mark as so") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) val toSchedule = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.periodicProcessService.deploy(toSchedule).futureValue - val deploymentEntity = f.repository.deploymentEntities.loneElement + val deploymentEntity = f.manager.deploymentEntities.loneElement deploymentEntity.status shouldBe PeriodicProcessDeploymentStatus.Deployed ConfigFactory - .parseString(f.jarManagerStub.lastDeploymentWithJarData.value.inputConfigDuringExecutionJson) + .parseString(f.periodicDeploymentHandlerStub.lastDeploymentWithRuntimeParams.value.inputConfigDuringExecutionJson) .getString("runAt") shouldBe deploymentEntity.runAt.toString - val expectedDetails = createPeriodicProcessDeployment(f.repository.processEntities.loneElement, deploymentEntity) + val expectedDetails = + createPeriodicProcessDeployment(f.manager.processEntities.loneElement, deploymentEntity) f.events.toList shouldBe List(DeployedEvent(expectedDetails, None)) } test("deploy - should handle failed deployment") { val f = new Fixture - f.repository.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) - f.jarManagerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) + f.manager.addActiveProcess(processName, PeriodicProcessDeploymentStatus.Scheduled) + f.periodicDeploymentHandlerStub.deployWithJarFuture = Future.failed(new RuntimeException("Flink deploy error")) val toSchedule = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.periodicProcessService.deploy(toSchedule).futureValue - f.repository.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.FailedOnDeploy + f.manager.deploymentEntities.loneElement.status shouldBe PeriodicProcessDeploymentStatus.FailedOnDeploy val expectedDetails = createPeriodicProcessDeployment( - f.repository.processEntities.loneElement, - f.repository.deploymentEntities.loneElement + f.manager.processEntities.loneElement, + f.manager.deploymentEntities.loneElement, ) f.events.toList shouldBe List(FailedOnDeployEvent(expectedDetails, None)) } @@ -403,7 +420,7 @@ class PeriodicProcessServiceTest def tryToSchedule(schedule: ScheduleProperty): Unit = f.periodicProcessService - .schedule(schedule, ProcessVersion.empty, canonicalProcess, randomProcessActionId) + .schedule(schedule, processVersion, randomProcessActionId) .futureValue tryToSchedule(cronInFuture) shouldBe (()) @@ -470,9 +487,9 @@ class PeriodicProcessServiceTest val f = new Fixture val now = LocalDateTime.now() val scheduleProperty = MultipleScheduleProperty(Map("schedule1" -> cronInPast, "schedule2" -> cronInFuture)) - val periodicProcessId = f.repository.addOnlyProcess(processName, scheduleProperty) + val periodicProcessId = f.manager.addOnlyProcess(processName, scheduleProperty) statuses.zip(schedules).zipWithIndex.foreach { case ((status, schedule), index) => - f.repository.addOnlyDeployment( + f.manager.addOnlyDeployment( periodicProcessId, status = status, runAt = now.plusDays(index), diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala similarity index 82% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala index 05002ea839e..8ab5777f028 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/PeriodicProcessStateDefinitionManagerTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/PeriodicProcessStateDefinitionManagerTest.scala @@ -1,17 +1,10 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} -import pl.touk.nussknacker.engine.management.periodic.PeriodicProcessStateDefinitionManager.statusTooltip -import pl.touk.nussknacker.engine.management.periodic.model.{ - PeriodicProcessDeploymentId, - PeriodicProcessDeploymentState, - PeriodicProcessDeploymentStatus, - PeriodicProcessId, - ScheduleId, - ScheduleName -} +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessService.{DeploymentStatus, PeriodicProcessStatus} +import pl.touk.nussknacker.engine.common.periodic.PeriodicProcessStateDefinitionManager.statusTooltip import java.time.LocalDateTime import java.util.concurrent.atomic.AtomicLong diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala similarity index 91% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala index 720364a78d4..f92a0959d8d 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/RescheduleFinishedActorTest.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/RescheduleFinishedActorTest.scala @@ -1,10 +1,11 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.ActorSystem import akka.testkit.{TestKit, TestKitBase, TestProbe} import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.BeforeAndAfterAll +import pl.touk.nussknacker.engine.common.periodic.RescheduleFinishedActor import scala.concurrent.Future import scala.concurrent.duration._ diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala similarity index 94% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala index a6d53da9c16..c539f6c8085 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/UtilsSpec.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/UtilsSpec.scala @@ -1,4 +1,4 @@ -package pl.touk.nussknacker.engine.management.periodic +package pl.touk.nussknacker.engine.management.periodic.flink import akka.actor.{Actor, ActorSystem, Props} import akka.testkit.TestKit @@ -6,7 +6,8 @@ import com.typesafe.scalalogging.LazyLogging import org.scalatest.BeforeAndAfterAll import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpecLike -import pl.touk.nussknacker.engine.management.periodic.Utils.createActorWithRetry +import pl.touk.nussknacker.engine.common.periodic.Utils +import pl.touk.nussknacker.engine.common.periodic.Utils.createActorWithRetry import scala.concurrent.duration.Duration import scala.concurrent.{Await, Future} diff --git a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala similarity index 57% rename from engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala rename to engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala index d16786058ea..ab541b8e7a1 100644 --- a/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/db/InMemPeriodicProcessesRepository.scala +++ b/engine/flink/management/periodic/src/test/scala/pl/touk/nussknacker/engine/management/periodic/flink/db/InMemPeriodicProcessesManager.scala @@ -1,19 +1,17 @@ -package pl.touk.nussknacker.engine.management.periodic.db +package pl.touk.nussknacker.engine.management.periodic.flink.db -import cats.{Id, Monad} import io.circe.syntax.EncoderOps +import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.ProcessActionId +import pl.touk.nussknacker.engine.api.deployment.periodic.PeriodicProcessesManager +import pl.touk.nussknacker.engine.api.deployment.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus +import pl.touk.nussknacker.engine.api.deployment.periodic.model._ import pl.touk.nussknacker.engine.api.process.{ProcessName, VersionId} import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.management.periodic._ -import pl.touk.nussknacker.engine.management.periodic.db.InMemPeriodicProcessesRepository.{ - DeploymentIdSequence, - ProcessIdSequence -} -import pl.touk.nussknacker.engine.management.periodic.db.PeriodicProcessesRepository.createPeriodicProcessDeployment -import pl.touk.nussknacker.engine.management.periodic.model.PeriodicProcessDeploymentStatus.PeriodicProcessDeploymentStatus -import pl.touk.nussknacker.engine.management.periodic.model._ +import pl.touk.nussknacker.engine.common.periodic.ScheduleProperty.{fromApi, toApi} +import pl.touk.nussknacker.engine.common.periodic.{CronScheduleProperty, ScheduleProperty, SingleScheduleProperty} +import pl.touk.nussknacker.engine.management.periodic.flink.db.InMemPeriodicProcessesManager._ import java.time.chrono.ChronoLocalDateTime import java.time.{LocalDateTime, ZoneId} @@ -23,23 +21,19 @@ import scala.collection.mutable.ListBuffer import scala.concurrent.Future import scala.util.Random -object InMemPeriodicProcessesRepository { - private val ProcessIdSequence = new AtomicLong(0) - private val DeploymentIdSequence = new AtomicLong(0) -} - -class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicProcessesRepository { +class InMemPeriodicProcessesManager(processingType: String) extends PeriodicProcessesManager { var processEntities: mutable.ListBuffer[PeriodicProcessEntity] = ListBuffer.empty var deploymentEntities: mutable.ListBuffer[PeriodicProcessDeploymentEntity] = ListBuffer.empty - private implicit val localDateOrdering: Ordering[LocalDateTime] = Ordering.by(identity[ChronoLocalDateTime[_]]) - - override type Action[T] = Id[T] - - override implicit def monad: Monad[Id] = cats.catsInstancesForId + private def canonicalProcess(processName: ProcessName) = { + ScenarioBuilder + .streaming(processName.value) + .source("start", "source") + .emptySink("end", "KafkaSink") + } - override def run[T](action: Id[T]): Future[T] = Future.successful(action) + private implicit val localDateOrdering: Ordering[LocalDateTime] = Ordering.by(identity[ChronoLocalDateTime[_]]) def addActiveProcess( processName: ProcessName, @@ -73,14 +67,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP processName = processName, processVersionId = VersionId.initialVersionId, processingType = processingType, - processJson = Some( - ScenarioBuilder - .streaming(processName.value) - .source("start", "source") - .emptySink("end", "KafkaSink") - ), inputConfigDuringExecutionJson = "{}", - jarFileName = "", + runtimeParams = RuntimeParams(Map("jarFileName" -> "")), scheduleProperty = scheduleProperty.asJson.noSpaces, active = true, createdAt = LocalDateTime.now(), @@ -117,46 +105,46 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getSchedulesState( scenarioName: ProcessName - ): Action[SchedulesState] = { + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities.filter { pe => pe.processName == scenarioName && deploymentEntities.exists(d => d.periodicProcessId == pe.id) }.toSeq getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount = Int.MaxValue) } - override def markInactive(processId: PeriodicProcessId): Unit = + override def markInactive(processId: PeriodicProcessId): Future[Unit] = Future.successful { processEntities.zipWithIndex .find { case (process, _) => process.id == processId } .foreach { case (process, index) => processEntities.update(index, process.copy(active = false)) } + } override def create( - deploymentWithJarData: DeploymentWithJarData[CanonicalProcess], - scheduleProperty: ScheduleProperty, + deploymentWithRuntimeParams: DeploymentWithRuntimeParams, + scheduleProperty: PeriodicProcessesManager.ScheduleProperty, processActionId: ProcessActionId, - ): PeriodicProcess[CanonicalProcess] = { + ): Future[PeriodicProcess] = Future.successful { val id = PeriodicProcessId(Random.nextLong()) val periodicProcess = PeriodicProcessEntity( id = id, - processName = deploymentWithJarData.processVersion.processName, - processVersionId = deploymentWithJarData.processVersion.versionId, + processName = deploymentWithRuntimeParams.processVersion.processName, + processVersionId = deploymentWithRuntimeParams.processVersion.versionId, processingType = processingType, - processJson = Some(deploymentWithJarData.process), - inputConfigDuringExecutionJson = deploymentWithJarData.inputConfigDuringExecutionJson, - jarFileName = deploymentWithJarData.jarFileName, - scheduleProperty = scheduleProperty.asJson.noSpaces, + inputConfigDuringExecutionJson = deploymentWithRuntimeParams.inputConfigDuringExecutionJson, + runtimeParams = deploymentWithRuntimeParams.runtimeParams, + scheduleProperty = fromApi(scheduleProperty).asJson.noSpaces, active = true, createdAt = LocalDateTime.now(), processActionId = Some(processActionId) ) processEntities += periodicProcess - PeriodicProcessesRepository.createPeriodicProcessWithJson(periodicProcess) + createPeriodicProcess(periodicProcess) } override def findActiveSchedulesForProcessesHavingDeploymentWithMatchingStatus( - expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus] - ): Action[SchedulesState] = { + expectedDeploymentStatuses: Set[PeriodicProcessDeploymentStatus], + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities.filter { pe => pe.processingType == processingType && deploymentEntities.exists(d => d.periodicProcessId == pe.id && expectedDeploymentStatuses.contains(d.status)) @@ -166,18 +154,19 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP override def getLatestDeploymentsForActiveSchedules( processName: ProcessName, - deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = Future.successful { getLatestDeploymentsForPeriodicProcesses( processEntities(processName).filter(_.active), deploymentsPerScheduleMaxCount ) + } override def getLatestDeploymentsForLatestInactiveSchedules( processName: ProcessName, inactiveProcessesMaxCount: Int, - deploymentsPerScheduleMaxCount: Int - ): Action[SchedulesState] = { + deploymentsPerScheduleMaxCount: Int, + ): Future[SchedulesState] = Future.successful { val filteredProcesses = processEntities(processName).filterNot(_.active).sortBy(_.createdAt).takeRight(inactiveProcessesMaxCount) getLatestDeploymentsForPeriodicProcesses(filteredProcesses, deploymentsPerScheduleMaxCount) @@ -197,43 +186,41 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP val ds = deployments .sortBy(d => -d.runAt.atZone(ZoneId.systemDefault()).toInstant.toEpochMilli) .take(deploymentsPerScheduleMaxCount) - .map(ScheduleDeploymentData(_)) + .map(scheduleDeploymentData(_)) .toList - scheduleId -> ScheduleData(PeriodicProcessesRepository.createPeriodicProcessWithoutJson(process), ds) + scheduleId -> ScheduleData(createPeriodicProcess(process), ds) } } yield deploymentGroupedByScheduleName).toMap) - override def findToBeDeployed: Seq[PeriodicProcessDeployment[CanonicalProcess]] = { + override def findToBeDeployed: Future[Seq[PeriodicProcessDeployment]] = { val scheduled = findActive(PeriodicProcessDeploymentStatus.Scheduled) readyToRun(scheduled) } - override def findToBeRetried: Action[Seq[PeriodicProcessDeployment[CanonicalProcess]]] = { + override def findToBeRetried: Future[Seq[PeriodicProcessDeployment]] = { val toBeRetried = findActive(PeriodicProcessDeploymentStatus.FailedOnDeploy).filter(_.retriesLeft > 0) readyToRun(toBeRetried) } - override def findProcessData(id: PeriodicProcessDeploymentId): PeriodicProcessDeployment[CanonicalProcess] = + override def findProcessData( + id: PeriodicProcessDeploymentId, + ): Future[PeriodicProcessDeployment] = Future.successful { (for { d <- deploymentEntities if d.id == id p <- processEntities if p.id == d.periodicProcessId } yield createPeriodicProcessDeployment(p, d)).head - - override def findProcessData(processName: ProcessName): Seq[PeriodicProcess[CanonicalProcess]] = - processEntities(processName) - .filter(_.active) - .map(PeriodicProcessesRepository.createPeriodicProcessWithJson) + } private def processEntities(processName: ProcessName): Seq[PeriodicProcessEntity] = processEntities .filter(process => process.processName == processName && process.processingType == processingType) .toSeq - override def markDeployed(id: PeriodicProcessDeploymentId): Unit = { + override def markDeployed(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)(_.copy(status = PeriodicProcessDeploymentStatus.Deployed, deployedAt = Some(LocalDateTime.now()))) } - override def markFinished(id: PeriodicProcessDeploymentId): Unit = { + override def markFinished(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)(_.copy(status = PeriodicProcessDeploymentStatus.Finished, completedAt = Some(LocalDateTime.now()))) } @@ -242,7 +229,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP status: PeriodicProcessDeploymentStatus, deployRetries: Int, retryAt: Option[LocalDateTime] - ): Action[Unit] = { + ): Future[Unit] = Future.successful { update(id)( _.copy( status = status, @@ -253,7 +240,7 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP ) } - override def markFailed(id: PeriodicProcessDeploymentId): Unit = { + override def markFailed(id: PeriodicProcessDeploymentId): Future[Unit] = Future.successful { update(id)( _.copy( status = PeriodicProcessDeploymentStatus.Failed, @@ -266,8 +253,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP id: PeriodicProcessId, scheduleName: ScheduleName, runAt: LocalDateTime, - deployMaxRetries: Int - ): PeriodicProcessDeployment[CanonicalProcess] = { + deployMaxRetries: Int, + ): Future[PeriodicProcessDeployment] = Future.successful { val deploymentEntity = PeriodicProcessDeploymentEntity( id = PeriodicProcessDeploymentId(Random.nextLong()), periodicProcessId = id, @@ -281,7 +268,8 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP status = PeriodicProcessDeploymentStatus.Scheduled ) deploymentEntities += deploymentEntity - createPeriodicProcessDeployment(processEntities.find(_.id == id).head, deploymentEntity) + val processEntity = processEntities.find(_.id == id).head + createPeriodicProcessDeployment(processEntity, deploymentEntity) } private def update( @@ -294,24 +282,129 @@ class InMemPeriodicProcessesRepository(processingType: String) extends PeriodicP } } - private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment[CanonicalProcess]] = + private def findActive(status: PeriodicProcessDeploymentStatus): Seq[PeriodicProcessDeployment] = findActive( Seq(status) ) private def findActive( statusList: Seq[PeriodicProcessDeploymentStatus] - ): Seq[PeriodicProcessDeployment[CanonicalProcess]] = + ): Seq[PeriodicProcessDeployment] = (for { p <- processEntities if p.active && p.processingType == processingType d <- deploymentEntities if d.periodicProcessId == p.id && statusList.contains(d.status) } yield createPeriodicProcessDeployment(p, d)).toSeq private def readyToRun( - deployments: Seq[PeriodicProcessDeployment[CanonicalProcess]] - ): Seq[PeriodicProcessDeployment[CanonicalProcess]] = { + deployments: Seq[PeriodicProcessDeployment] + ): Future[Seq[PeriodicProcessDeployment]] = { val now = LocalDateTime.now() - deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now)) + Future.successful(deployments.filter(d => d.runAt.isBefore(now) || d.runAt.isEqual(now))) + } + + override def fetchCanonicalProcess( + processName: ProcessName, + versionId: VersionId + ): Future[Option[CanonicalProcess]] = Future.successful(Some(canonicalProcess(processName))) + +} + +object InMemPeriodicProcessesManager { + + private val ProcessIdSequence = new AtomicLong(0) + private val DeploymentIdSequence = new AtomicLong(0) + + final case class PeriodicProcessEntity( + id: PeriodicProcessId, + processName: ProcessName, + processVersionId: VersionId, + processingType: String, + inputConfigDuringExecutionJson: String, + runtimeParams: RuntimeParams, + scheduleProperty: String, + active: Boolean, + createdAt: LocalDateTime, + processActionId: Option[ProcessActionId] + ) + + case class PeriodicProcessDeploymentEntity( + id: PeriodicProcessDeploymentId, + periodicProcessId: PeriodicProcessId, + createdAt: LocalDateTime, + runAt: LocalDateTime, + scheduleName: Option[String], + deployedAt: Option[LocalDateTime], + completedAt: Option[LocalDateTime], + retriesLeft: Int, + nextRetryAt: Option[LocalDateTime], + status: PeriodicProcessDeploymentStatus + ) + + def createPeriodicProcessDeployment( + processEntity: PeriodicProcessEntity, + processDeploymentEntity: PeriodicProcessDeploymentEntity, + ): PeriodicProcessDeployment = { + val process = createPeriodicProcess(processEntity) + PeriodicProcessDeployment( + processDeploymentEntity.id, + process, + processDeploymentEntity.createdAt, + processDeploymentEntity.runAt, + ScheduleName(processDeploymentEntity.scheduleName), + processDeploymentEntity.retriesLeft, + processDeploymentEntity.nextRetryAt, + createPeriodicDeploymentState(processDeploymentEntity) + ) + } + + def createPeriodicDeploymentState( + processDeploymentEntity: PeriodicProcessDeploymentEntity + ): PeriodicProcessDeploymentState = { + PeriodicProcessDeploymentState( + processDeploymentEntity.deployedAt, + processDeploymentEntity.completedAt, + processDeploymentEntity.status + ) + } + + def createPeriodicProcess(processEntity: PeriodicProcessEntity): PeriodicProcess = { + val processVersion = createProcessVersion(processEntity) + val scheduleProperty = prepareScheduleProperty(processEntity) + PeriodicProcess( + processEntity.id, + DeploymentWithRuntimeParams( + processVersion = processVersion, + inputConfigDuringExecutionJson = processEntity.inputConfigDuringExecutionJson, + runtimeParams = processEntity.runtimeParams, + ), + scheduleProperty, + processEntity.active, + processEntity.createdAt, + processEntity.processActionId + ) + } + + private def prepareScheduleProperty(processEntity: PeriodicProcessEntity) = { + val scheduleProperty = io.circe.parser + .decode[ScheduleProperty](processEntity.scheduleProperty) + .fold(e => throw new IllegalArgumentException(e), identity) + toApi(scheduleProperty) + } + + private def createProcessVersion(processEntity: PeriodicProcessEntity): ProcessVersion = { + ProcessVersion.empty.copy(versionId = processEntity.processVersionId, processName = processEntity.processName) + } + + private def scheduleDeploymentData(deployment: PeriodicProcessDeploymentEntity): ScheduleDeploymentData = { + ScheduleDeploymentData( + deployment.id, + deployment.createdAt, + deployment.runAt, + deployment.deployedAt, + deployment.retriesLeft, + deployment.nextRetryAt, + createPeriodicDeploymentState(deployment) + ) } } diff --git a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala index 931b35d5794..4a5f1202cf9 100644 --- a/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala +++ b/engine/flink/management/src/it/scala/pl/touk/nussknacker/engine/management/streaming/FlinkStreamingDeploymentManagerProviderHelper.scala @@ -3,23 +3,17 @@ package pl.touk.nussknacker.engine.management.streaming import akka.actor.ActorSystem import org.asynchttpclient.DefaultAsyncHttpClientConfig import pl.touk.nussknacker.engine.api.component.DesignerWideComponentId +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.{ DeploymentManager, NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager + ProcessingTypeDeployedScenariosProviderStub } import pl.touk.nussknacker.engine.definition.component.Components.ComponentDefinitionExtractionMode import pl.touk.nussknacker.engine.management.FlinkStreamingDeploymentManagerProvider -import pl.touk.nussknacker.engine.{ - ConfigWithUnresolvedVersion, - DeploymentManagerDependencies, - ModelData, - ModelDependencies, - ProcessingTypeConfig -} -import sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend +import pl.touk.nussknacker.engine._ +import _root_.sttp.client3.asynchttpclient.future.AsyncHttpClientFutureBackend object FlinkStreamingDeploymentManagerProviderHelper { @@ -43,6 +37,7 @@ object FlinkStreamingDeploymentManagerProviderHelper { new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, actorSystem.dispatcher, actorSystem, backend diff --git a/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala b/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala index f379f73891b..722764a7496 100644 --- a/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala +++ b/engine/flink/management/src/test/scala/pl/touk/nussknacker/engine/management/FlinkRestManagerSpec.scala @@ -14,18 +14,13 @@ import pl.touk.nussknacker.engine.api.component.NodesDeploymentData import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.deployment.inconsistency.InconsistentStateDetector +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId} import pl.touk.nussknacker.engine.api.{MetaData, ProcessVersion, StreamMetaData} import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess -import pl.touk.nussknacker.engine.deployment.{ - AdditionalModelConfigs, - DeploymentData, - DeploymentId, - ExternalDeploymentId, - User -} +import pl.touk.nussknacker.engine.deployment._ import pl.touk.nussknacker.engine.management.rest.HttpFlinkClient import pl.touk.nussknacker.engine.management.rest.flinkRestModel._ import pl.touk.nussknacker.engine.testing.LocalModelData @@ -553,6 +548,7 @@ class FlinkRestManagerSpec extends AnyFunSuite with Matchers with PatientScalaFu new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, ExecutionContext.global, ActorSystem(getClass.getSimpleName), sttpBackend diff --git a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala index 0d28fae5775..fc846926fd5 100644 --- a/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala +++ b/engine/lite/embeddedDeploymentManager/src/test/scala/pl/touk/nussknacker/streaming/embedded/RequestResponseEmbeddedDeploymentManagerTest.scala @@ -8,19 +8,9 @@ import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy import pl.touk.nussknacker.engine.api.deployment.cache.ScenarioStateCachingConfig +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus -import pl.touk.nussknacker.engine.api.deployment.{ - DMCancelScenarioCommand, - DMRunDeploymentCommand, - DataFreshnessPolicy, - DeployedScenarioData, - DeploymentManager, - DeploymentUpdateStrategy, - NoOpScenarioActivityManager, - ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager -} +import pl.touk.nussknacker.engine.api.deployment._ import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.build.ScenarioBuilder import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess @@ -56,6 +46,7 @@ class RequestResponseEmbeddedDeploymentManagerTest new ProcessingTypeDeployedScenariosProviderStub(initiallyDeployedScenarios), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, as.dispatcher, as, SttpBackendStub.asynchronousFuture diff --git a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala index 05b006824bc..4de62eb847d 100644 --- a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala +++ b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/BaseK8sDeploymentManagerTest.scala @@ -9,8 +9,9 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import pl.touk.nussknacker.engine.api.ProcessVersion import pl.touk.nussknacker.engine.api.deployment.DeploymentUpdateStrategy.StateRestoringStrategy -import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.api.deployment._ +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider +import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess import pl.touk.nussknacker.engine.deployment.DeploymentData import pl.touk.nussknacker.engine.{DeploymentManagerDependencies, ModelData} @@ -61,6 +62,7 @@ class BaseK8sDeploymentManagerTest new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, system.dispatcher, system, backend diff --git a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala index 16b645f8582..f1837cc3a05 100644 --- a/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala +++ b/engine/lite/k8sDeploymentManager/src/test/scala/pl/touk/nussknacker/k8s/manager/K8sDeploymentManagerOnMocksTest.scala @@ -9,13 +9,12 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.scalatest.{BeforeAndAfterAll, Inside, OptionValues} import pl.touk.nussknacker.engine.DeploymentManagerDependencies +import pl.touk.nussknacker.engine.api.deployment.periodic.NoOpPeriodicProcessesManagerProvider import pl.touk.nussknacker.engine.api.deployment.{ DataFreshnessPolicy, NoOpScenarioActivityManager, ProcessingTypeActionServiceStub, - ProcessingTypeDeployedScenariosProvider, - ProcessingTypeDeployedScenariosProviderStub, - ScenarioActivityManager + ProcessingTypeDeployedScenariosProviderStub } import pl.touk.nussknacker.engine.api.process.ProcessName import pl.touk.nussknacker.engine.testing.LocalModelData @@ -75,6 +74,7 @@ class K8sDeploymentManagerOnMocksTest new ProcessingTypeDeployedScenariosProviderStub(List.empty), new ProcessingTypeActionServiceStub, NoOpScenarioActivityManager, + NoOpPeriodicProcessesManagerProvider, system.dispatcher, system, SttpBackendStub.asynchronousFuture