Merge branch 'release/v0.3'

Dockerfile

@@ -1,7 +1,7 @@
 FROM python:3.5
 
 # Apt-install mysql client and cleanup temporary files afterwards
-RUN apt-get update && apt-get install -y mysql-client git vim && apt-get clean
+RUN apt-get update && apt-get install -y mysql-client git vim liblua5.1-dev libmagickwand-dev && apt-get clean
 RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 COPY requirements.txt /tmp/requirements.txt

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include static *

requirements.txt

@@ -18,5 +18,7 @@ trueskill
 aiocron
 marisa-trie
 oauth2client
-git+https://github.com/FAForever/faftools.git@develop#egg=faftools
-pyopenssl
+git+https://github.com/FAForever/faftools.git@38fc8e10f3cbd50d3f6abe9168e9040c08ad26cf#egg=faftools
+pyopenssl
+mock
+lupa

server/__init__.py

@@ -26,7 +26,7 @@
 from .ladder_service import LadderService
 from .control import init as run_control_server
 
-__version__ = '0.2'
+__version__ = '0.3'
 __author__ = 'Chris Kitching, Dragonfire, Gael Honorez, Jeroen De Dauw, Crotalus, Michael Søndergaard, Michel Jung'
 __contact__ = '[email protected]'
 __license__ = 'GPLv3'

server/api/api_accessor.py

@@ -1,10 +1,14 @@
 import json
 from functools import partial
 import asyncio
+
+import pkg_resources
 from httplib2 import Http
 from oauth2client.service_account import ServiceAccountCredentials
 from server.config import API_TOKEN_URI, API_BASE_URL
 
+CACERTS_FILE = pkg_resources.resource_filename('static', 'cacerts.txt')
+
 
 class ApiAccessor:
     def __init__(self):
@@ -25,10 +29,13 @@ async def api_post(self, path, player_id, data=None, headers=None):
         loop = asyncio.get_event_loop()
         result = await loop.run_in_executor(
             None,
-            partial(self.http(player_id).request, API_BASE_URL + path, "POST", headers=headers, body=json.dumps(data)))
+            partial(self.http(player_id).request, API_BASE_URL + path, "POST", headers=headers, body=json.dumps(data))
+        )
 
         return result
 
     def http(self, sub=None):
         credentials = self._service_account_credentials.create_delegated(sub)
-        return credentials.authorize(Http())
+        # FIXME ca_certs=CACERTS_FILE should be used, but it didn't work for some reason.
+        # Since we'll access the API locally over HTTP in future anyway, I decided to just skip validation for now
+        return credentials.authorize(Http(disable_ssl_certificate_validation=True))

server/config.py

@@ -11,7 +11,9 @@
 
 logging.getLogger('aiomeasures').setLevel(logging.INFO)
 
-trueskill.setup(mu=1500, sigma=500, beta=250, tau=5, draw_probability=0.10)
+# Credit to Axle for parameter changes, see: http://forums.faforever.com/viewtopic.php?f=45&t=11698#p119599
+# Optimum values for ladder here, using them for global as well.
+trueskill.setup(mu=1500, sigma=500, beta=240, tau=10, draw_probability=0.10)
 
 STATSD_SERVER = os.getenv('STATSD_SERVER', '127.0.0.1:8125')
 
@@ -48,5 +50,5 @@
 
 API_CLIENT_ID = os.getenv("API_CLIENT_ID", "6ccaf75b-a1f3-48be-bac3-4e9ffba81eb7")
 API_CLIENT_SECRET = os.getenv("API_CLIENT_SECRET", "banana")
-API_TOKEN_URI = os.getenv("API_TOKEN_URI", "http://api.dev.faforever.com/jwt/auth")
-API_BASE_URL = os.getenv("API_BASE_URL", "http://api.dev.faforever.com/jwt")
+API_TOKEN_URI = os.getenv("API_TOKEN_URI", "https://api.dev.faforever.com/jwt/auth")
+API_BASE_URL = os.getenv("API_BASE_URL", "https://api.dev.faforever.com/jwt")

server/connectivity.py

@@ -223,7 +223,7 @@ async def test_public(self):
         for i in range(0, 3):
             await self.send_natpacket(self.remote_addr, message)
         try:
-            result = await asyncio.wait_for(received_packet, 1)
+            result = await asyncio.wait_for(received_packet, 10)
             self._logger.debug("Result: {}".format(result))
             return True
         except (CancelledError, TimeoutError):
@@ -241,7 +241,7 @@ async def test_stun(self):
                                          message])
         await asyncio.sleep(0.1)
         try:
-            received, addr = await asyncio.wait_for(future, 20.0)
+            received, addr = await asyncio.wait_for(future, 60.0)
             if received == message:
                 delta = time.time() - start_time
                 self._logger.debug("{} replied from {} in {}".format(self.identifier, addr, delta))

server/game_service.py

@@ -169,7 +169,6 @@ def create_game(self,
             game = Game(**args)
         self.games[id] = game
 
-        self._logger.debug("{} created".format(game))
         game.visibility = visibility
         game.password = password
 

server/gameconnection.py

@@ -337,7 +337,7 @@ async def handle_action(self, command, args):
             elif command == 'OperationComplete':
                 if int(args[0]) == 1:
                     secondary, delta = int(args[1]), str(args[2])
-                    with await db.db_pool.get() as conn:
+                    async with db.db_pool.get() as conn:
                         cursor = await conn.cursor()
                         # FIXME: Resolve used map earlier than this
                         await cursor.execute("SELECT id FROM coop_map WHERE filename LIKE '%/"
@@ -358,6 +358,23 @@ async def handle_action(self, command, args):
                 self.game.enforce_rating = True
 
 
+            elif command == 'TeamkillReport':
+                # args[0] -> seconds of gametime when kill happened
+                # args[1] -> victim id
+                # args[2] -> victim nickname (for debug purpose only)
+                # args[3] -> teamkiller id
+                # args[3] -> teamkiller nickname (for debug purpose only)
+                gametime, victim_id, victim_name, teamkiller_id, teamkiller_name = args
+
+                async with db.db_pool.get() as conn:
+                    cursor = await conn.cursor()
+
+                    await cursor.execute("INSERT INTO `teamkills`"
+                                         "(`teamkiller`, `victim`, `game_id`, `gametime`) "
+                                         "VALUES (%s, %s, %s, %s);",
+                                         (teamkiller_id, victim_id, self.game.id, gametime))
+
+
         except AuthenticationError as e:
             self.log.exception("Authentication error: {}".format(e))
             self.abort()

server/games/game.py

@@ -118,7 +118,6 @@ def __init__(self, id, game_service, game_stats_service,
         self.map_scenario_path = None
         self.password = None
         self._players = []
-        self.gameType = 0
         self.AIs = {}
         self.desyncs = 0
         self.validity = ValidityState.VALID
@@ -136,11 +135,12 @@ def __init__(self, id, game_service, game_stats_service,
 
         self.mods = {}
         self._logger.debug("{} created".format(self))
-        asyncio.get_event_loop().call_later(20, self.timeout_game)
+        asyncio.get_event_loop().create_task(self.timeout_game())
 
-    def timeout_game(self):
+    async def timeout_game(self):
+        await asyncio.sleep(20)
         if self.state == GameState.INITIALIZING:
-            self.state = GameState.ENDED
+            await self.on_game_end()
 
     @property
     def armies(self):
@@ -495,15 +495,13 @@ async def update_game_stats(self):
 
             # Determine if the map is blacklisted, and invalidate the game for ranking purposes if
             # so, and grab the map id at the same time.
-            await cursor.execute("SELECT table_map.id as map_id, table_map_unranked.id as unranked "
-                                 "FROM table_map LEFT JOIN table_map_unranked "
-                                 "ON table_map.id = table_map_unranked.id "
-                                 "WHERE table_map.filename = %s", (self.map_file_path,))
+            await cursor.execute("SELECT id, ranked FROM map_version "
+                                 "WHERE filename = %s", (self.map_file_path,))
             result = await cursor.fetchone()
             if result:
-                (self.map_id, blacklist_flag) = result
+                (self.map_id, ranked) = result
 
-                if blacklist_flag:
+                if not ranked:
                     await self.mark_invalid(ValidityState.BAD_MAP)
 
             modId = self.game_service.featured_mods[self.game_mode].id
@@ -662,7 +660,6 @@ def to_dict(self):
             "map_file_path": self.map_file_path,
             "host": self.host.login if self.host else '',
             "num_players": len(self.players),
-            "game_type": self.gameType,
             "max_players": self.max_players,
             "launched_at": self.launched_at,
             "teams": {

server/lobbyconnection.py

@@ -511,7 +511,7 @@ async def check_user_login(self, cursor, login, password):
         if dbPassword != password:
             raise AuthenticationError("Login not found or password incorrect. They are case sensitive.")
 
-        if ban_reason is not None and ban_expiry is not None and datetime.datetime.now() < ban_expiry:
+        if ban_reason is not None and datetime.datetime.now() < ban_expiry:
             raise ClientError("You are banned from FAF.\n Reason :\n {}".format(ban_reason))
 
         self._logger.debug("Login from: {}, {}, {}".format(player_id, login, self.session))
@@ -746,16 +746,29 @@ async def command_hello(self, message):
                 url, tooltip = avatar
                 self.player.avatar = {"url": url, "tooltip": tooltip}
 
-        self.sendJSON(dict(command="welcome", id=self.player.id, login=login))
+        # Send the player their own player info.
+        self.sendJSON({
+            "command": "welcome",
+            "me": self.player.to_dict(),
+
+            # For backwards compatibility for old clients. For now.
+            "id": self.player.id,
+            "login": login
+        })
 
-        # Tell player about everybody online
+        # Tell player about everybody online. This must happen after "welcome".
         self.sendJSON(
             {
                 "command": "player_info",
                 "players": [player.to_dict() for player in self.player_service]
             }
         )
-        # Tell everyone else online about us
+
+        # Tell everyone else online about us. This must happen after all the player_info messages.
+        # This ensures that no other client will perform an operation that interacts with the
+        # incoming user, allowing the client to make useful assumptions: it can be certain it has
+        # initialised its local player service before it is going to get messages that want to
+        # query it.
         self.player_service.mark_dirty(self.player)
 
         friends = []

server/player_service.py

@@ -107,7 +107,7 @@ def is_uniqueid_exempt(self, user_id):
     def has_blacklisted_domain(self, email):
         # A valid email only has one @ anyway.
         domain = email.split("@")[1]
-        return len(self.blacklisted_email_domains.keys(domain[::-1])) != 0
+        return domain in self.blacklisted_email_domains
 
     def get_player(self, player_id):
         if player_id in self.players:
@@ -140,4 +140,4 @@ async def update_data(self):
             rows = await cursor.fetchall()
             # Get list of reversed blacklisted domains (so we can (pre)suffix-match incoming emails
             # in sublinear time)
-            self.blacklisted_email_domains = marisa_trie.Trie(map(lambda x: x[0][::-1], rows))
+            self.blacklisted_email_domains = marisa_trie.Trie(map(lambda x: x[0], rows))

server/stats/game_stats_service.py

@@ -42,7 +42,8 @@ async def process_game_stats(self, player: Player, game: Game, stats_json):
         a_queue = []
         # Stores events to batch update
         e_queue = []
-        survived = stats['units']['cdr'].get('lost', 0) == 0
+        survived = stats['units']['cdr'].get('lost', 0) < stats['units']['cdr'].get('built', 1)
+        blueprint_stats = stats['blueprints']
         unit_stats = stats['units']
 
         if survived and game.game_mode == 'ladder1v1':
@@ -54,30 +55,30 @@ async def process_game_stats(self, player: Player, game: Game, stats_json):
         self._increment(ACH_VETERAN, 1, a_queue)
         self._increment(ACH_ADDICT, 1, a_queue)
 
-        self._category_stats(unit_stats, survived, a_queue, e_queue)
         self._faction_played(faction, survived, a_queue, e_queue)
-        self._killed_acus(unit_stats['cdr'].get('kills', 0), survived, a_queue)
-        self._built_mercies(self._count(unit_stats, lambda x: x.get('built', 0), Unit.MERCY), a_queue)
-        self._built_fire_beetles(self._count(unit_stats, lambda x: x.get('built', 0), Unit.FIRE_BEETLE), a_queue)
-        self._built_salvations(self._count(unit_stats, lambda x: x.get('built', 0), Unit.SALVATION), survived, a_queue)
-        self._built_yolona_oss(self._count(unit_stats, lambda x: x.get('built', 0), Unit.YOLONA_OSS), survived, a_queue)
-        self._built_paragons(self._count(unit_stats, lambda x: x.get('built', 0), Unit.PARAGON), survived, a_queue)
-        self._built_atlantis(self._count(unit_stats, lambda x: x.get('built', 0), Unit.ATLANTIS), a_queue)
-        self._built_tempests(self._count(unit_stats, lambda x: x.get('built', 0), Unit.TEMPEST), a_queue)
-        self._built_scathis(self._count(unit_stats, lambda x: x.get('built', 0), Unit.SCATHIS), survived, a_queue)
-        self._built_mavors(self._count(unit_stats, lambda x: x.get('built', 0), Unit.MAVOR), survived, a_queue)
-        self._built_czars(self._count(unit_stats, lambda x: x.get('built', 0), Unit.CZAR), a_queue)
-        self._built_ahwassas(self._count(unit_stats, lambda x: x.get('built', 0), Unit.AHWASSA), a_queue)
-        self._built_ythothas(self._count(unit_stats, lambda x: x.get('built', 0), Unit.YTHOTHA), a_queue)
-        self._built_fatboys(self._count(unit_stats, lambda x: x.get('built', 0), Unit.FATBOY), a_queue)
-        self._built_monkeylords(self._count(unit_stats, lambda x: x.get('built', 0), Unit.MONKEYLORD), a_queue)
-        self._built_galactic_colossus(self._count(unit_stats, lambda x: x.get('built', 0), Unit.GALACTIC_COLOSSUS), a_queue)
-        self._built_soul_rippers(self._count(unit_stats, lambda x: x.get('built', 0), Unit.SOUL_RIPPER), a_queue)
-        self._built_megaliths(self._count(unit_stats, lambda x: x.get('built', 0), Unit.MEGALITH), a_queue)
-        self._built_asfs(self._count(unit_stats, lambda x: x.get('built', 0), *ASFS), a_queue)
+        self._category_stats(unit_stats, survived, a_queue, e_queue)
+        self._killed_acus(unit_stats, survived, a_queue)
+        self._built_mercies(_count_built_units(blueprint_stats, Unit.MERCY), a_queue)
+        self._built_fire_beetles(_count_built_units(blueprint_stats, Unit.FIRE_BEETLE), a_queue)
+        self._built_salvations(_count_built_units(blueprint_stats, Unit.SALVATION), survived, a_queue)
+        self._built_yolona_oss(_count_built_units(blueprint_stats, Unit.YOLONA_OSS), survived, a_queue)
+        self._built_paragons(_count_built_units(blueprint_stats, Unit.PARAGON), survived, a_queue)
+        self._built_atlantis(_count_built_units(blueprint_stats, Unit.ATLANTIS), a_queue)
+        self._built_tempests(_count_built_units(blueprint_stats, Unit.TEMPEST), a_queue)
+        self._built_scathis(_count_built_units(blueprint_stats, Unit.SCATHIS), survived, a_queue)
+        self._built_mavors(_count_built_units(blueprint_stats, Unit.MAVOR), survived, a_queue)
+        self._built_czars(_count_built_units(blueprint_stats, Unit.CZAR), a_queue)
+        self._built_ahwassas(_count_built_units(blueprint_stats, Unit.AHWASSA), a_queue)
+        self._built_ythothas(_count_built_units(blueprint_stats, Unit.YTHOTHA), a_queue)
+        self._built_fatboys(_count_built_units(blueprint_stats, Unit.FATBOY), a_queue)
+        self._built_monkeylords(_count_built_units(blueprint_stats, Unit.MONKEYLORD), a_queue)
+        self._built_galactic_colossus(_count_built_units(blueprint_stats, Unit.GALACTIC_COLOSSUS), a_queue)
+        self._built_soul_rippers(_count_built_units(blueprint_stats, Unit.SOUL_RIPPER), a_queue)
+        self._built_megaliths(_count_built_units(blueprint_stats, Unit.MEGALITH), a_queue)
+        self._built_asfs(_count_built_units(blueprint_stats, *ASFS), a_queue)
         self._built_transports(unit_stats['transportation'].get('built', 0), a_queue)
         self._built_sacus(unit_stats['sacu'].get('built', 0), a_queue)
-        self._lowest_acu_health(self._count(unit_stats, lambda x: x.get('lowest_health', 0), *ACUS), survived, a_queue)
+        self._lowest_acu_health(_count(blueprint_stats, lambda x: x.get('lowest_health', 0), *ACUS), survived, a_queue)
 
         updated_achievements = await self._achievement_service.execute_batch_update(player.id, a_queue)
         await self._event_service.execute_batch_update(player.id, e_queue)
@@ -164,11 +165,16 @@ def _faction_played(self, faction, survived, achievements_queue, events_queue):
                 self._increment(ACH_YENZYNE, 1, achievements_queue)
                 self._increment(ACH_SUTHANUS, 1, achievements_queue)
 
-    def _killed_acus(self, count, survived, achievements_queue):
-        if count >= 3 and survived:
+    def _killed_acus(self, unit_stats, survived, achievements_queue):
+        acus_per_player = unit_stats['cdr'].get('built', 1)
+        killed_acus = unit_stats['cdr'].get('kills', 0)
+
+        # I'm aware that this is not perfectly correct, but it's edge case and tracking who got killed by whom would
+        # require game code changes
+        if killed_acus >= 3 * acus_per_player and survived:
             self._unlock(ACH_HATTRICK, achievements_queue)
 
-        self._increment(ACH_DONT_MESS_WITH_ME, count, achievements_queue)
+        self._increment(ACH_DONT_MESS_WITH_ME, int(killed_acus / acus_per_player), achievements_queue)
 
     def _built_mercies(self, count, achievements_queue):
         self._increment(ACH_NO_MERCY, count, achievements_queue)
@@ -251,11 +257,15 @@ def _set_steps_at_least(self, achievement_id, steps, achievements_queue):
     def _record_event(self, event_id, count, events_queue):
         self._event_service.record_event(event_id, count, events_queue)
 
-    @staticmethod
-    def _count(unit_stats, function, *units):
-        result = 0
-        for unit in units:
-            if unit.value in unit_stats:
-                result += function(unit_stats[unit.value])
 
-        return result
+def _count_built_units(stats, *units):
+    return _count(stats, lambda x: x.get('built', 0), *units)
+
+
+def _count(stats, function, *units):
+    result = 0
+    for unit in units:
+        if unit.value in stats:
+            result += function(stats[unit.value])
+
+    return result

setup.py

@@ -1,14 +1,17 @@
 from distutils.core import setup
 
+from setuptools import find_packages
+
 import server
 
 setup(
     name='Forged Alliance Forever Server',
     version=server.__version__,
-    packages=['server'],
+    packages=['server'] + find_packages(),
     url='http://www.faforever.com',
     license=server.__license__,
     author=server.__author__,
     author_email=server.__contact__,
-    description='Lobby/game server project'
+    description='Lobby/game server project',
+    include_package_data=True
 )

static/__init__.py

@@ -0,0 +1,3 @@
+"""
+Contains static data like CA certificates.
+"""