From a98b7ab6f32a14f8301bdafc6c8fc9ecc1806e28 Mon Sep 17 00:00:00 2001 From: Evgeniia Date: Sun, 16 Jan 2022 01:07:02 +0100 Subject: [PATCH] Bug fixes (#414) * start web app * fix * do not set loop in web * remove uvloop * get event loop * setup web and api * catch correct exc * update tanner docker file and set env --- docker/docker-compose.yml | 8 +++++++- docker/tanner/Dockerfile | 13 +++++++++---- requirements.txt | 1 - tanner/api/api.py | 8 ++++---- tanner/api/server.py | 10 +++++----- tanner/server.py | 17 ++++++----------- tanner/web/server.py | 8 ++++---- 7 files changed, 35 insertions(+), 30 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 82c80e3f..d222d7f9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -33,6 +33,8 @@ services: # Tanner API Service tanner_api: build: ./tanner + environment: + - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner_api restart: always stop_signal: SIGKILL @@ -44,13 +46,15 @@ services: - tanner_local image: "mushorg/tanner:latest" read_only: true - command: tannerapi + command: tannerapi depends_on: - tanner_redis # Tanner WEB Service tanner_web: build: ./tanner + environment: + - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner_web restart: always stop_signal: SIGKILL @@ -71,6 +75,8 @@ services: # Tanner Service tanner: build: ./tanner + environment: + - PATH=/opt/tanner/tanner-env/bin/:$PATH container_name: tanner restart: always stop_signal: SIGKILL diff --git a/docker/tanner/Dockerfile b/docker/tanner/Dockerfile index 786d5db6..f6551f80 100644 --- a/docker/tanner/Dockerfile +++ b/docker/tanner/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.10 +FROM alpine:3.15 # Include dist ADD dist/ /root/dist/ @@ -13,12 +13,16 @@ RUN apk -U --no-cache add \ linux-headers \ py3-yarl \ python3 \ - python3-dev && \ + python3-dev \ + py3-pip && \ # Setup Tanner git clone --depth=1 https://github.com/mushorg/tanner /opt/tanner && \ cp /root/dist/config.yaml /opt/tanner/tanner/data/ && \ cd /opt/tanner/ && \ - pip3 install --no-cache-dir setuptools && \ + python3 -m venv tanner-env && \ + source /opt/tanner/tanner-env/bin/activate && \ + pip install --upgrade pip && \ + pip3 install --no-cache-dir wheel && \ pip3 install --no-cache-dir -r requirements.txt && \ python3 setup.py install && \ cd / && \ @@ -36,4 +40,5 @@ RUN apk -U --no-cache add \ # Start tanner USER nobody:nobody WORKDIR /opt/tanner -CMD tanner --config /opt/tanner/tanner/data/config.yaml +CMD ["/opt/tanner/tanner-env/bin/tanner", "--config", "/opt/tanner/tanner/data/config.yaml"] + diff --git a/requirements.txt b/requirements.txt index b7423b88..ed0e3901 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ mimesis<3.0.0 yarl redis aioredis -uvloop pymongo pylibinjection jinja2 diff --git a/tanner/api/api.py b/tanner/api/api.py index 3cab6f87..725dc899 100644 --- a/tanner/api/api.py +++ b/tanner/api/api.py @@ -12,8 +12,8 @@ def __init__(self, redis_client): async def return_snares(self): query_res = [] try: - query_res = await self.redis_client.smembers("snare_ids", encoding="utf-8") - except aioredis.ProtocolError as connection_error: + query_res = await self.redis_client.smembers("snare_ids") + except aioredis.exceptions.ConnectionError as connection_error: self.logger.exception("Can not connect to redis %s", connection_error) return list(query_res) @@ -39,8 +39,8 @@ async def return_snare_stats(self, snare_uuid): async def return_snare_info(self, uuid, count=-1): query_res = [] try: - query_res = await self.redis_client.zrevrangebyscore(uuid, offset=0, count=count, encoding="utf-8") - except aioredis.ProtocolError as connection_error: + query_res = await self.redis_client.zrevrangebyscore(uuid, offset=0, count=count) + except aioredis.exceptions.ConnectionError as connection_error: self.logger.exception("Can not connect to redis %s", connection_error) else: if not query_res: diff --git a/tanner/api/server.py b/tanner/api/server.py index 49b07460..571921bc 100644 --- a/tanner/api/server.py +++ b/tanner/api/server.py @@ -94,11 +94,11 @@ def setup_routes(self, app): app.router.add_resource("/{snare_uuid}/sessions").add_route("GET", self.handle_sessions) app.router.add_resource("/session/{sess_uuid}").add_route("GET", self.handle_session_info) - def create_app(self, loop, auth=False): + async def make_app(self, auth=False): if auth: - app = web.Application(loop=loop, middlewares=[self.auth]) + app = web.Application(middlewares=[self.auth]) else: - app = web.Application(loop=loop) + app = web.Application() app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) return app @@ -108,7 +108,7 @@ def start(self): self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) self.api = api.Api(self.redis_client) set_auth = TannerConfig.get("API", "auth") - app = self.create_app(loop, set_auth) + host = TannerConfig.get("API", "host") port = int(TannerConfig.get("API", "port")) @@ -116,4 +116,4 @@ def start(self): key = generate() print("API_KEY for full access:", key) - web.run_app(app, host=host, port=port) + web.run_app(self.make_app(auth=set_auth), host=host, port=port) diff --git a/tanner/server.py b/tanner/server.py index da36e3a0..d15e9bfc 100644 --- a/tanner/server.py +++ b/tanner/server.py @@ -1,7 +1,6 @@ import asyncio import json import logging -import uvloop import yarl from aiohttp import web @@ -15,9 +14,6 @@ from tanner.reporting.log_hpfeeds import Reporting as hpfeeds_report from tanner import __version__ as tanner_version -asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) - - class TannerServer: def __init__(self): base_dir = TannerConfig.get("EMULATORS", "root_dir") @@ -113,10 +109,12 @@ def setup_routes(self, app): app.router.add_get("/dorks", self.handle_dorks) app.router.add_get("/version", self.handle_version) - def create_app(self, loop): - app = web.Application(loop=loop) + async def make_app(self): + app = web.Application() app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) + app.on_startup.append(self.start_background_delete) + app.on_cleanup.append(self.cleanup_background_tasks) return app async def start_background_delete(self, app): @@ -130,10 +128,7 @@ def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client()) - app = self.create_app(loop) - app.on_startup.append(self.start_background_delete) - app.on_cleanup.append(self.cleanup_background_tasks) - host = TannerConfig.get("TANNER", "host") port = TannerConfig.get("TANNER", "port") - web.run_app(app, host=host, port=int(port)) + + web.run_app(self.make_app(), host=host, port=port) diff --git a/tanner/web/server.py b/tanner/web/server.py index 8098811c..3c94b6bc 100644 --- a/tanner/web/server.py +++ b/tanner/web/server.py @@ -94,8 +94,8 @@ def setup_routes(self, app): app.router.add_resource("/{snare_uuid}/sessions/page/{page_id}").add_route("GET", self.handle_sessions) app.router.add_static("/static/", path="tanner/web/static") - def create_app(self, loop): - app = web.Application(loop=loop) + async def make_app(self): + app = web.Application() aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader("tanner/web/templates")) app.on_shutdown.append(self.on_shutdown) self.setup_routes(app) @@ -105,7 +105,7 @@ def start(self): loop = asyncio.get_event_loop() self.redis_client = loop.run_until_complete(redis_client.RedisClient.get_redis_client(poolsize=20)) self.api = api.Api(self.redis_client) - app = self.create_app(loop) + host = TannerConfig.get("WEB", "host") port = int(TannerConfig.get("WEB", "port")) - web.run_app(app, host=host, port=port) + web.run_app(self.make_app(), host=host, port=port)