Reload Configuration Variables (#570)

* Manage configuration variables inside an object. * Service that manages refreshing. * Read config info from conf.yaml * Update README.md * Callback on config variable change. * Restart control server on port change * Reload geoip file path * Tests for refreshing geoip file path * Add log level callback * Profiler class * Address review comments * Make ice_server default variables refresh correctly * Review comments * Move default config into python file to give mypy an easier time. * Add some tests and isort some imports * Fix readme link * Revert item access to ConfigStore * Handle default values with attribute syntax * Remove unused import * Minor review comments * Log config variable changes individually * Treat empty or absent config files properly * Re-use a single profiler object so it can be canceled later * Run profiler on COUNT or DURATION change * pipenv lock * Revert "pipenv lock" This reverts commit 173872b. Co-authored-by: Askaholic <[email protected]>
FAForever · May 4, 2020 · 6ec63d4 · 6ec63d4
1 parent 890d151
commit 6ec63d4
Show file tree

Hide file tree

Showing 30 changed files with 740 additions and 201 deletions.
diff --git a/Pipfile b/Pipfile
@@ -22,6 +22,7 @@ sqlalchemy = "*"
 twilio = "*"
 humanize = "*"
 aiomysql = {editable = true, git = "https://github.com/aio-libs/aiomysql"}
+pyyaml = "*"
 
 [dev-packages]
 pytest = "*"

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -29,9 +29,21 @@ Check if the container is running with
 
 If you cannot find `faf-server` in the list, run `docker run` without `-d` to see what happens.
 
-If you have a different root password, database name then the default (see [config.py](https://github.com/FAForever/server/blob/develop/server/config.py#L43)), you must pass it over the environment parameter of docker, e.g.
+## Configuration
 
-    docker run --link faf-db:db -p 8001:8001 -p 30351:30351 -e FAF_DB_PASSWORD=<wanted_password> -e FAF_DB_NAME=<db_name> faf-server
+If you have for example a different root password or database name than the default
+`DB_PASSWORD` and `DB_NAME` entries in
+[config.py](https://github.com/FAForever/server/blob/develop/server/config.py),
+you should provide a custom configuration file.
+This file will be used for all variables that it defines
+while the default values of `config.py` still apply for those it doesn't.
+To use your custom configuration file, pass its location as an environment
+variable to docker:
+
+    docker run --link faf-db:db -p 8001:8001 -p 30351:30351 -e CONFIGURATION_FILE=<path> faf-server
+
+You can find an example configuration file under
+[tests/data/test_config.yaml](https://github.com/FAForever/server/blob/develop/tests/data/test_config.yaml).
 
 # Contributing
 

diff --git a/server.py b/server.py
@@ -17,15 +17,12 @@
 from datetime import datetime
 
 import server
-import server.config as config
+from server.config import config
 from docopt import docopt
 from server.api.api_accessor import ApiAccessor
-from server.config import (
-    DB_LOGIN, DB_NAME, DB_PASSWORD, DB_PORT, DB_SERVER, TWILIO_ACCOUNT_SID
-)
 from server.core import create_services
 from server.ice_servers.nts import TwilioNTS
-from server.timing import at_interval
+from server.profiler import Profiler
 
 
 async def main():
@@ -44,32 +41,27 @@ def signal_handler(sig: int, _frame):
     signal.signal(signal.SIGTERM, signal_handler)
     signal.signal(signal.SIGINT, signal_handler)
 
-    if config.ENABLE_METRICS:
-        logger.info("Using prometheus on port: {}".format(config.METRICS_PORT))
-
     database = server.db.FAFDatabase(loop)
     await database.connect(
-        host=DB_SERVER,
-        port=int(DB_PORT),
-        user=DB_LOGIN,
-        password=DB_PASSWORD,
+        host=config.DB_SERVER,
+        port=int(config.DB_PORT),
+        user=config.DB_LOGIN,
+        password=config.DB_PASSWORD,
         maxsize=10,
-        db=DB_NAME,
+        db=config.DB_NAME,
     )
 
     # Set up services
 
     twilio_nts = None
-    if TWILIO_ACCOUNT_SID:
+    if config.TWILIO_ACCOUNT_SID:
         twilio_nts = TwilioNTS()
     else:
         logger.warning(
             "Twilio is not set up. You must set TWILIO_ACCOUNT_SID and TWILIO_TOKEN to use the Twilio ICE servers."
         )
 
-    api_accessor = None
-    if config.USE_API:
-        api_accessor = ApiAccessor()
+    api_accessor = ApiAccessor()
 
     services = create_services({
         "api_accessor": api_accessor,
@@ -81,38 +73,28 @@ def signal_handler(sig: int, _frame):
         service.initialize() for service in services.values()
     ])
 
-    if config.PROFILING_INTERVAL > 0:
-        logger.warning("Profiling enabled! This will create additional load.")
-        import cProfile
-        pr = cProfile.Profile()
-        profiled_count = 0
-        max_count = 300
-
-        @at_interval(config.PROFILING_INTERVAL, loop=loop)
-        async def run_profiler():
-            nonlocal profiled_count
-            nonlocal pr
-
-            if len(services["player_service"]) > 1000:
-                return
-            elif profiled_count >= max_count:
-                del pr
-                return
-
-            logger.info("Starting profiler")
-            pr.enable()
-            await asyncio.sleep(2)
-            pr.disable()
-            profiled_count += 1
-
-            logging.info("Done profiling %i/%i", profiled_count, max_count)
-            pr.dump_stats("profile.txt")
+    profiler = Profiler(services["player_service"])
+    profiler.refresh()
+    config.register_callback("PROFILING_COUNT", profiler.refresh)
+    config.register_callback("PROFILING_DURATION", profiler.refresh)
+    config.register_callback("PROFILING_INTERVAL", profiler.refresh)
 
     ctrl_server = await server.run_control_server(
         services["player_service"],
         services["game_service"]
     )
 
+    async def restart_control_server():
+        nonlocal ctrl_server
+        nonlocal services
+
+        await ctrl_server.shutdown()
+        ctrl_server = await server.run_control_server(
+            services["player_service"],
+            services["game_service"]
+        )
+    config.register_callback("CONTROL_SERVER_PORT", restart_control_server)
+
     lobby_server = await server.run_lobby_server(
         address=('', 8001),
         database=database,

diff --git a/server/__init__.py b/server/__init__.py
@@ -12,14 +12,15 @@
 from prometheus_client import start_http_server
 
 from server.db import FAFDatabase
-from . import config as config
+from .config import config
 from .games.game import GameState, VisibilityState
 from .stats.game_stats_service import GameStatsService
 from .gameconnection import GameConnection
 from .ice_servers.nts import TwilioNTS
 from .lobbyconnection import LobbyConnection
 from .protocol import QDataStreamProtocol
 from .servercontext import ServerContext
+from .configuration_service import ConfigurationService  # noqa: F401
 from .geoip_service import GeoIpService
 from .player_service import PlayerService
 from .game_service import GameService
@@ -51,6 +52,7 @@
 logger = logging.getLogger("server")
 
 if config.ENABLE_METRICS:
+    logger.info("Using prometheus on port: %i", config.METRICS_PORT)
     start_http_server(config.METRICS_PORT)
 
 

diff --git a/server/api/api_accessor.py b/server/api/api_accessor.py
@@ -4,9 +4,7 @@
 from oauthlib.oauth2.rfc6749.errors import (
     InsecureTransportError, MissingTokenError
 )
-from server.config import (
-    API_BASE_URL, API_CLIENT_ID, API_CLIENT_SECRET, API_TOKEN_URI
-)
+from server.config import config
 from server.decorators import with_logger
 
 from .oauth_session import OAuth2Session
@@ -23,9 +21,9 @@ def __init__(self):
     async def get_session(self) -> Optional[OAuth2Session]:
         if not self.session:
             self.session = OAuth2Session(
-                client_id=API_CLIENT_ID,
-                client_secret=API_CLIENT_SECRET,
-                token_url=API_TOKEN_URI
+                client_id=config.API_CLIENT_ID,
+                client_secret=config.API_CLIENT_SECRET,
+                token_url=config.API_TOKEN_URI
             )
         if not self.session.is_expired():
             return self.session
@@ -43,8 +41,8 @@ async def get_session(self) -> Optional[OAuth2Session]:
         except InsecureTransportError:  # pragma: no cover
             self._logger.error(
                 "API (%s,%s) should be HTTPS, not HTTP. Enable OAUTHLIB_INSECURE_TRANSPORT to avoid this warning.",
-                API_BASE_URL,
-                API_TOKEN_URI
+                config.API_BASE_URL,
+                config.API_TOKEN_URI
             )
         except SSLError:  # pragma: no cover
             self._logger.error("The certificate verification failed while connecting the API")
@@ -86,14 +84,14 @@ async def update_events(self, events_data, player_id):
 
     async def api_get(self, path):
         api = await self.api_session.get_session()
-        return await api.request('GET', API_BASE_URL + path)
+        return await api.request('GET', config.API_BASE_URL + path)
 
     async def api_patch(self, path, json_data):
         api = await self.api_session.get_session()
         headers = {'Content-type': 'application/json'}
         status, data = await api.request(
             "PATCH",
-            API_BASE_URL + path,
+            config.API_BASE_URL + path,
             headers=headers,
             json=json_data
         )

diff --git a/server/api/oauth_session.py b/server/api/oauth_session.py
@@ -1,4 +1,3 @@
-import asyncio
 import os
 import time
 from typing import Dict