diff --git a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/Main.scala b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/Main.scala index d34dce5fcd..3ade19e782 100644 --- a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/Main.scala +++ b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/Main.scala @@ -16,6 +16,7 @@ import weco.pipeline.id_minter.config.builders.{ IdentifiersTableBuilder, RDSBuilder } +import weco.pipeline.id_minter.config.models.RDSClientConfig import weco.pipeline.id_minter.database.IdentifiersDao import weco.pipeline.id_minter.models.IdentifiersTable import weco.pipeline.id_minter.services.IdMinterWorkerService @@ -29,9 +30,9 @@ object Main extends WellcomeTypesafeApp { config: Config => implicit val executionContext: ExecutionContext = ActorSystem("main-actor-system").dispatcher - + val rdsConfig = RDSClientConfig(config) val identifiersTableConfig = IdentifiersTableBuilder.buildConfig(config) - RDSBuilder.buildDB(config) + RDSBuilder.buildDB(rdsConfig) val identifierGenerator = new IdentifierGenerator( identifiersDao = new IdentifiersDao( @@ -66,7 +67,7 @@ object Main extends WellcomeTypesafeApp { identifierGenerator = identifierGenerator, jsonRetriever = jsonRetriever, pipelineStream = pipelineStream, - rdsClientConfig = RDSBuilder.buildRDSClientConfig(config), + rdsClientConfig = rdsConfig, identifiersTableConfig = identifiersTableConfig ) } diff --git a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/builders/RDSBuilder.scala b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/builders/RDSBuilder.scala index 5aeb5db56a..94632f0614 100644 --- a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/builders/RDSBuilder.scala +++ b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/builders/RDSBuilder.scala @@ -1,25 +1,13 @@ package weco.pipeline.id_minter.config.builders -import com.typesafe.config.Config import scalikejdbc.{ConnectionPool, ConnectionPoolSettings} import weco.pipeline.id_minter.config.models.RDSClientConfig -import weco.typesafe.config.builders.EnrichConfig._ object RDSBuilder { - def buildDB(config: Config): Unit = { - val maxConnections = config.requireInt("aws.rds.maxConnections") - val rdsClientConfig = buildRDSClientConfig(config) - - buildDB( - maxConnections = maxConnections, - rdsClientConfig = rdsClientConfig - ) - } - - def buildDB(maxConnections: Int, rdsClientConfig: RDSClientConfig): Unit = { + def buildDB(rdsClientConfig: RDSClientConfig): Unit = { val connectionPoolSettings = ConnectionPoolSettings( - maxSize = maxConnections, + maxSize = rdsClientConfig.maxConnections, connectionTimeoutMillis = 120000L ) @@ -41,23 +29,4 @@ object RDSBuilder { ) } - def buildRDSClientConfig(config: Config): RDSClientConfig = { - val primaryHost = config.requireString("aws.rds.primary_host") - val replicaHost = config.requireString("aws.rds.replica_host") - - val port = config - .getIntOption("aws.rds.port") - .getOrElse(3306) - - val username = config.requireString("aws.rds.username") - val password = config.requireString("aws.rds.password") - - RDSClientConfig( - primaryHost = primaryHost, - replicaHost = replicaHost, - port = port, - username = username, - password = password - ) - } } diff --git a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/models/RDSClientConfig.scala b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/models/RDSClientConfig.scala index e6de7e88e5..7a5c22fada 100644 --- a/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/models/RDSClientConfig.scala +++ b/pipeline/id_minter/src/main/scala/weco/pipeline/id_minter/config/models/RDSClientConfig.scala @@ -1,9 +1,36 @@ package weco.pipeline.id_minter.config.models +import com.typesafe.config.Config +import weco.typesafe.config.builders.EnrichConfig._ + case class RDSClientConfig( primaryHost: String, replicaHost: String, port: Int, username: String, - password: String + password: String, + maxConnections: Int ) + +object RDSClientConfig { + def apply(config: Config): RDSClientConfig = { + val primaryHost = config.requireString("aws.rds.primary_host") + val replicaHost = config.requireString("aws.rds.replica_host") + + val port = config + .getIntOption("aws.rds.port") + .getOrElse(3306) + + val username = config.requireString("aws.rds.username") + val password = config.requireString("aws.rds.password") + + RDSClientConfig( + primaryHost = primaryHost, + replicaHost = replicaHost, + port = port, + username = username, + password = password, + maxConnections = config.requireInt("aws.rds.maxConnections") + ) + } +} diff --git a/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/config/builders/RDSBuilderTest.scala b/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/config/builders/RDSBuilderTest.scala new file mode 100644 index 0000000000..4c5f4feaa0 --- /dev/null +++ b/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/config/builders/RDSBuilderTest.scala @@ -0,0 +1,84 @@ +package weco.pipeline.id_minter.config.builders + +import com.typesafe.config.ConfigFactory +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import scalikejdbc.ConnectionPool +import weco.pipeline.id_minter.config.models.RDSClientConfig + +import scala.jdk.CollectionConverters._ + +class RDSBuilderTest extends AnyFunSpec with Matchers { + describe("Building the database client connection pools") { + it("sets up a connection pool for the primary host") { + RDSBuilder.buildDB( + RDSClientConfig( + primaryHost = "realdeal.example.com", + replicaHost = "doppelganger.example.com", + port = 999, + username = "slartibartfast", + password = "ssh_its_a_secret", + maxConnections = 5 + ) + ) + val p = ConnectionPool.get(name = 'primary) + p.url shouldBe "jdbc:mysql://realdeal.example.com:999" + p.user shouldBe "slartibartfast" + p.settings.maxSize shouldBe 5 + } + + it("sets up a connection pool for the replica host") { + RDSBuilder.buildDB( + RDSClientConfig( + primaryHost = "primary.example.com", + replicaHost = "doppelganger.example.com", + port = 1234, + username = "slartibartfast", + password = "ssh_its_a_secret", + maxConnections = 12 + ) + ) + val p = ConnectionPool.get(name = 'replica) + p.url shouldBe "jdbc:mysql://doppelganger.example.com:1234" + p.user shouldBe "slartibartfast" + p.settings.maxSize shouldBe 12 + } + } + describe("Extracting configuration values") { + val rawConfig = Map( + "aws.rds.primary_host" -> "numberone", + "aws.rds.replica_host" -> "numbertwo", + "aws.rds.username" -> "rincewind", + "aws.rds.password" -> "opensesame", + "aws.rds.maxConnections" -> "678" + ) + val config = RDSClientConfig( + ConfigFactory.parseMap(rawConfig.asJava) + ) + it("extracts the primary host") { + config.primaryHost shouldBe "numberone" + } + it("extracts the replica host") { + config.replicaHost shouldBe "numbertwo" + } + it("extracts the username") { + config.username shouldBe "rincewind" + } + it("extracts the password") { + config.password shouldBe "opensesame" + } + it("extracts the maxConnections") { + config.maxConnections shouldBe 678 + } + it("defaults to port 3306") { + config.port shouldBe 3306 + } + it("extracts the port value if specified") { + val configWithPort = RDSClientConfig( + ConfigFactory.parseMap((rawConfig + ("aws.rds.port" -> 1234)).asJava) + ) + configWithPort.port shouldBe 1234 + + } + } +} diff --git a/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/fixtures/IdentifiersDatabase.scala b/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/fixtures/IdentifiersDatabase.scala index f018d0d0a5..f991124b0d 100644 --- a/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/fixtures/IdentifiersDatabase.scala +++ b/pipeline/id_minter/src/test/scala/weco/pipeline/id_minter/fixtures/IdentifiersDatabase.scala @@ -83,7 +83,8 @@ trait IdentifiersDatabase replicaHost = rdsHost, port = rdsPort, username = rdsUsername, - password = rdsPassword + password = rdsPassword, + maxConnections = rdsMaxPoolSize ) def withIdentifiersTable[R](