prediction data from priv api

.github/workflows/hetzner-production-workflow.yml

@@ -1,30 +1,25 @@
 name: hetzner production workflow
 
-# Controls when the workflow will run
-# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows
 on:
-  # Triggers the workflow on push or pull request events but only for the "master" branch
   push:
-    branches: [ develop ]
+    branches:
+      - develop
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
-permissions: write-all
-
-
 env:
   VALUE_FILE: bd-ml-prd/deployment.yaml
   REGISTRY: quay.io/bot_detector/bd-ml
-
+  ENVIRONMENT: prd
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
-  # This workflow contains a single job called "build"
+  # This workflow contains two jobs: build_image and update_image_version
   build_image:
     # The type of runner that the job will run on
-    runs-on: [self-hosted, hetzner]
-
+    runs-on: [self-hosted, "hetzner"]
+    if: github.repository_owner == 'Bot-detector'
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
@@ -35,35 +30,52 @@ jobs:
         id: vars
         run: |
           echo "GIT_HASH=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_OUTPUT
-          echo "REGISTRY=$REGISTRY" >> $GITHUB_OUTPUT
-          echo "VALUE_FILE=$VALUE_FILE" >> $GITHUB_OUTPUT
-      # Runs a command using the runners shell
-      - name: docker build
-        run: docker build . --file Dockerfile --network=host -t "$REGISTRY:${{ steps.vars.outputs.GIT_HASH }}" --target production --build-arg api_port=6532 --build-arg root_path=/ml
 
-      - name: login to registry
-        run: echo "${{ secrets.QUAY_REGISTERY_PASSWORD }}" | docker login -u="bot_detector+quay_robot" quay.io --password-stdin
+      - name: Docker Build
+        run: |
+          docker build . --file Dockerfile --network=host -t "${REGISTRY}:${{ steps.vars.outputs.GIT_HASH }}" --target production --build-arg api_port=5000 --build-arg root_path=""
+
+      - name: Login to Quay registry
+        run: echo "${{ secrets.QUAY_REGISTERY_PASSWORD }}" | docker login -u "bot_detector+quay_robot" quay.io --password-stdin
+
+      - name: Tag image
+        run: |
+          docker tag "${REGISTRY}:${{ steps.vars.outputs.GIT_HASH }}" "${REGISTRY}:${ENVIRONMENT}"
+
+      - name: Docker Push image to Quay registry
+        run: |
+          docker push "${REGISTRY}:${{ steps.vars.outputs.GIT_HASH }}"
+          docker push "${REGISTRY}:${ENVIRONMENT}"
 
-      - name: docker push image to registry
-        run: docker push "$REGISTRY:${{ steps.vars.outputs.GIT_HASH }}"
+  update_image_version:
+    runs-on: [self-hosted, "hetzner"]
+    if: github.repository_owner == 'Bot-detector'
+    needs: build_image # This ensures that the build_image job is completed before running this job
 
+    steps:
       - name: Checkout Target Repository
         uses: actions/checkout@v3
         with:
           repository: Bot-detector/bot-detector-k8s
 
+      - name: Set vars
+        id: vars
+        run: |
+          echo "GIT_HASH=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_OUTPUT
+
       - name: Update Image Version
         uses: fjogeleit/yaml-update-action@main
         with:
           repository: Bot-detector/bot-detector-k8s
-          valueFile: "${{ steps.vars.outputs.VALUE_FILE }}"
-          token: ${{ secrets.HETZNER_ACTIONS_RUNNER_TOKEN }}
+          valueFile: ${{ env.VALUE_FILE }}
+          token: "${{ secrets.HETZNER_ACTIONS_RUNNER_TOKEN }}"
           commitChange: true
-          branch: "${{ steps.vars.outputs.GIT_HASH }}"
+          title: "${{ env.VALUE_FILE }}_${{ steps.vars.outputs.GIT_HASH }}"
+          branch: "${{ env.VALUE_FILE }}_${{ steps.vars.outputs.GIT_HASH }}"
           targetBranch: develop
           masterBranchName: develop
           createPR: true
           changes: |
             {
-              "spec.template.spec.containers[0].image":"${{ steps.vars.outputs.REGISTRY }}:${{ steps.vars.outputs.GIT_HASH }}"
+              "spec.template.spec.containers[0].image": "${{ env.REGISTRY }}:${{ steps.vars.outputs.GIT_HASH }}"
             }

api/app.py

@@ -14,6 +14,7 @@
 from api.cogs import requests as req
 from api.MachineLearning import classifier, data
 import aiohttp
+from datetime import date
 
 app = config.app
 
@@ -61,10 +62,17 @@ async def manual_startup(secret: str):
     """
     # secret token for api's to talk to eachother
     if secret != config.secret_token:
-        raise HTTPException(status_code=404, detail=f"insufficient permissions")
+        raise HTTPException(status_code=404, detail="insufficient permissions")
 
+    id = 0
+    today = date.today()
     while True:
-        hiscores = await req.get_prediction_data()
+        if today != date.today():
+            logger.info("new day")
+            id, today = 0, date.today()
+
+        hiscores = await req.get_prediction_data(id=id, limit=config.BATCH_AMOUNT)
+        id = hiscores[-1].get("id")
         hiscores = pd.DataFrame(hiscores)
 
         if len(hiscores) == 0:
@@ -80,6 +88,11 @@ async def manual_startup(secret: str):
 
         logger.debug("Sending response")
         await req.post_prediction(output)
+
+        if len(hiscores) < config.BATCH_AMOUNT:
+            sleep=60
+            logger.info(f"{len(hiscores)=} < {config.BATCH_AMOUNT=}, sleeping: {sleep}")
+            await asyncio.sleep(sleep)
     return {"detail": "ok"}
 
 

api/cogs/requests.py

@@ -132,9 +132,14 @@ async def get_hiscore_data(label_id):
     return hiscores
 
 
-async def get_prediction_data():
+async def get_prediction_data(id:int=0, limit:int=0):
     url = f"{config.detector_api}/v1/prediction/data"
-    params = {"token": config.token, "limit": config.BATCH_AMOUNT}
+    url = "http://private-api-svc.bd-prd.svc:5000/v2/highscore/latest" #TODO: fix hardcoded
+    params = {
+        "player_id":id,
+        "many": True,
+        "limit": limit
+    }
 
     data = await retry_request(url=url, params=params)
     return data