Changing entry node -> organic node && entry node -> synthetic node, with render deployment compatibility

Taskfile.yml

@@ -15,15 +15,15 @@ tasks:
   control_node_dev:
     cmds:
       - LOCALHOST=true ENV_FILE=.vali.env python -m validator.control_node.src.main
-
-  query_node_dev:
-    cmds: 
-      - LOCALHOST=true ENV_FILE=.vali.env python -m validator.query_node.src.main
 
-  entry_node_dev:
+  synthetic_node_dev:
     cmds:
-      - LOCALHOST=true uvicorn validator.entry_node.src.server:app --reload --host 0.0.0.0 --port 8090 --env-file .vali.env
-
+      - LOCALHOST=true ENV_FILE=.vali.env python -m validator.synthetic_node.src.main
+
+  organic_node_dev:
+    cmds:
+      - LOCALHOST=true uvicorn validator.organic_node.src.server:app --reload --host 0.0.0.0 --port 8090 --env-file .vali.env
+
   weight_setter_dev:
     cmds:
       - LOCALHOST=true ENV_FILE=.vali.env python -m validator.control_node.src.set_weights.calculate_and_schedule_weights
@@ -33,16 +33,16 @@ tasks:
       # turns off the nodes so you can run them on the host
       # turns _on_ the proxies and dev tooling so you can access the infrastructure directly from the host
       - "
-        docker compose 
-          --env-file .vali.env 
-          --profile entry_node_profile 
-          up -d --build 
-          --scale entry_node=0 
-          --scale query_node=0 
-          --scale control_node=0 
-          --scale weight_setter=0 
+        docker compose
+          --env-file .vali.env
+          --profile organic_node_profile
+          up -d --build
+          --scale organic_node=0
+          --scale synthetic_node=0
+          --scale control_node=0
+          --scale weight_setter=0
           --scale redis-proxy=1
-          --scale adminer=1 
+          --scale adminer=1
           --scale postgres-proxy=1
           --scale redis-commander=1
       "
@@ -59,33 +59,33 @@ tasks:
   update_docker_fiber_version:
     cmds:
       - docker run --rm nineteen-control_node sh -c "pip install git+https://github.com/rayonlabs/[email protected] --upgrade"
-      - docker run --rm nineteen-entry_node sh -c "pip install git+https://github.com/rayonlabs/[email protected] --upgrade"
-      - docker run --rm nineteen-query_node sh -c "pip install git+https://github.com/rayonlabs/[email protected] --upgrade"
+      - docker run --rm nineteen-organic_node sh -c "pip install git+https://github.com/rayonlabs/[email protected] --upgrade"
+      - docker run --rm nineteen-synthetic_node sh -c "pip install git+https://github.com/rayonlabs/[email protected] --upgrade"
 
   createm:
     cmds:
       - bash start_miners.sh
 
   startup:
     cmds:
-      - docker compose --env-file .vali.env --profile entry_node_profile up -d --build
+      - docker compose --env-file .vali.env --profile organic_node_profile up -d --build
 
   shutdown:
     cmds:
-      - docker compose --env-file .vali.env --profile entry_node_profile down --remove-orphans
+      - docker compose --env-file .vali.env --profile organic_node_profile down --remove-orphans
 
   shutdown_and_delete:
     cmds:
-      - docker compose --env-file .vali.env --profile entry_node_profile down --remove-orphans -v
+      - docker compose --env-file .vali.env --profile organic_node_profile down --remove-orphans -v
 
   postgres:
     cmds:
-      - docker compose --env-file .vali.env --profile entry_node_profile exec -it postgresql psql -U$POSTGRES_USER $POSTGRES_DB
+      - docker compose --env-file .vali.env --profile organic_node_profile exec -it postgresql psql -U$POSTGRES_USER $POSTGRES_DB
 
   redis:
     cmds:
-      - docker compose --env-file .vali.env --profile entry_node_profile exec -it redis redis-cli
- 
+      - docker compose --env-file .vali.env --profile organic_node_profile exec -it redis redis-cli
+
   expose-postgres:
     cmds:
-      - docker compose -f docker-compose.yml --env-file .vali.env up -d --scale postgres-proxy=1 --scale adminer=1 
+      - docker compose -f docker-compose.yml --env-file .vali.env up -d --scale postgres-proxy=1 --scale adminer=1

config/grafana-validator-errors.json

@@ -50,7 +50,7 @@
             "uid": "P8E80F9AEF21F6940"
           },
           "editorMode": "builder",
-          "expr": "{service_name=~\"control_node|entry_node|query_node\"} | label_format level=detected_level | level =~ `(warn|error)` | line_format `{{.service_name }} - {{.code_filepath}}:{{.code_function }}(L{{.code_lineno}}) - {{__line__}}`",
+          "expr": "{service_name=~\"control_node|organic_node|synthetic_node\"} | label_format level=detected_level | level =~ `(warn|error)` | line_format `{{.service_name }} - {{.code_filepath}}:{{.code_function }}(L{{.code_lineno}}) - {{__line__}}`",
           "legendFormat": "",
           "queryType": "range",
           "refId": "A"
@@ -145,7 +145,7 @@
             "uid": "P8E80F9AEF21F6940"
           },
           "editorMode": "builder",
-          "expr": "sum by(service_name) (rate({service_name=~\"control_node|entry_node|query_node\"} | label_format level=detected_level | level =~ `(warn|error)` [1m])) * 60.0",
+          "expr": "sum by(service_name) (rate({service_name=~\"control_node|organic_node|synthetic_node\"} | label_format level=detected_level | level =~ `(warn|error)` [1m])) * 60.0",
           "legendFormat": "{{.service_name}}",
           "queryType": "range",
           "refId": "A"

config/grafana-validator-general.json

@@ -108,7 +108,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_query_node_query_streaming_synthetic_tokens",
+          "expr": "validator_synthetic_node_query_streaming_synthetic_tokens",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -209,7 +209,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_query_node_query_streaming_synthetic_tokens_per_sec",
+          "expr": "validator_synthetic_node_query_streaming_synthetic_tokens_per_sec",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -310,7 +310,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_entry_node_text_tokens",
+          "expr": "validator_organic_node_text_tokens",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -411,7 +411,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_entry_node_text_tokens_per_sec",
+          "expr": "validator_organic_node_text_tokens_per_sec",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "instant": false,
@@ -511,7 +511,7 @@
           "disableTextWrap": false,
           "editorMode": "builder",
           "exemplar": false,
-          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"entry_node\", http_target=\"/v1/chat/completions\"}[1m]) * 60",
+          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"organic_node\", http_target=\"/v1/chat/completions\"}[1m]) * 60",
           "format": "time_series",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
@@ -612,7 +612,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"entry_node\", http_target=\"/v1/text-to-image\"}[1m]) * 60",
+          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"organic_node\", http_target=\"/v1/text-to-image\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -711,7 +711,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"entry_node\", http_target=\"/v1/image-to-image\"}[1m]) * 60",
+          "expr": "rate(http_server_response_size_bytes_count{exported_job=\"organic_node\", http_target=\"/v1/image-to-image\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -811,7 +811,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "rate(validator_query_node_process_queries_total{exported_job=\"query_node\"}[$__rate_interval]) * 60",
+          "expr": "rate(validator_synthetic_node_process_queries_total{exported_job=\"synthetic_node\"}[$__rate_interval]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -907,7 +907,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_query_node_process_failed_queries_total",
+          "expr": "validator_synthetic_node_process_failed_queries_total",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -981,7 +981,7 @@
               "options": {
                 "mode": "exclude",
                 "names": [
-                  "validator_query_node_src_concurrent_queries_processing_ratio {__name__=\"validator_query_node_src_concurrent_queries_processing_ratio\", exported_job=\"query_node\", instance=\"otel-collector:9090\", job=\"otel-collector-services\"}"
+                  "validator_synthetic_node_src_concurrent_queries_processing_ratio {__name__=\"validator_synthetic_node_src_concurrent_queries_processing_ratio\", exported_job=\"synthetic_node\", instance=\"otel-collector:9090\", job=\"otel-collector-services\"}"
                 ],
                 "prefix": "All except:",
                 "readOnly": true
@@ -1028,7 +1028,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "builder",
-          "expr": "validator_query_node_src_concurrent_queries_processing_ratio",
+          "expr": "validator_synthetic_node_src_concurrent_queries_processing_ratio",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -1318,7 +1318,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "code",
-          "expr": "rate(validator_entry_node_image_success_total{exported_job=\"entry_node\"}[1m]) * 60",
+          "expr": "rate(validator_organic_node_image_success_total{exported_job=\"organic_node\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -1416,7 +1416,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "code",
-          "expr": "rate(validator_entry_node_image_error_total{exported_job=\"entry_node\"}[1m]) * 60",
+          "expr": "rate(validator_organic_node_image_error_total{exported_job=\"organic_node\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -1514,7 +1514,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "code",
-          "expr": "rate(validator_entry_node_text_success_total{exported_job=\"entry_node\"}[1m]) * 60",
+          "expr": "rate(validator_organic_node_text_success_total{exported_job=\"organic_node\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -1612,7 +1612,7 @@
           },
           "disableTextWrap": false,
           "editorMode": "code",
-          "expr": "rate(validator_entry_node_text_error_total{exported_job=\"entry_node\"}[1m]) * 60",
+          "expr": "rate(validator_organic_node_text_error_total{exported_job=\"organic_node\"}[1m]) * 60",
           "fullMetaSearch": false,
           "includeNullMetadata": false,
           "instant": false,
@@ -1654,7 +1654,7 @@
             "uid": "P8E80F9AEF21F6940"
           },
           "editorMode": "builder",
-          "expr": "{service_name=~\"control_node|entry_node|query_node\"} | label_format level=detected_level | line_format `{{.service_name }} - {{__line__}}`",
+          "expr": "{service_name=~\"control_node|organic_node|synthetic_node\"} | label_format level=detected_level | line_format `{{.service_name }} - {{__line__}}`",
           "legendFormat": "",
           "queryType": "range",
           "refId": "A"
@@ -1680,4 +1680,4 @@
   "uid": "ae1q8n928qc5ce",
   "version": 13,
   "weekStart": ""
-}
+}

core/constants.py

@@ -16,7 +16,5 @@
 # BASE_NINETEEN_API_URL = "http://localhost:8033/"
 BASE_NINETEEN_API_URL = "https://taovision.ai/"
 
-
-
 MIN_STEPS = "min_steps"
 MAX_STEPS = "max_steps"

docker-compose.yml

@@ -45,11 +45,11 @@ services:
       restart_policy:
         condition: on-failure
 
-  query_node:
-    container_name: query_node
+  synthetic_node:
+    container_name: synthetic_node
     build:
       context: .
-      dockerfile: dockerfiles/query_node.dockerfile
+      dockerfile: dockerfiles/synthetic_node.dockerfile
     deploy:
       replicas: 1
       resources:
@@ -63,7 +63,7 @@ services:
       - postgresql
     volumes:
       - ${HOME}/.bittensor/wallets:/root/.bittensor/wallets
-    entrypoint: ["opentelemetry-instrument", "--logs_exporter", "otlp", "--traces_exporter", "none", "--metrics_exporter", "otlp", "--service_name", "query_node", "python", "-u", "/app/validator/query_node/src/main.py"]
+    entrypoint: ["opentelemetry-instrument", "--logs_exporter", "otlp", "--traces_exporter", "none", "--metrics_exporter", "otlp", "--service_name", "synthetic_node", "python", "-u", "/app/validator/synthetic_node/src/main.py"]
     environment:
       OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
@@ -79,11 +79,10 @@ services:
     env_file:
       - .vali.env
 
-  entry_node:
-    container_name: entry_node
+  organic_node:
     build:
       context: .
-      dockerfile: dockerfiles/entry_node.dockerfile
+      dockerfile: dockerfiles/organic_node.dockerfile
     deploy:
       replicas: 1
       resources:
@@ -95,9 +94,12 @@ services:
     depends_on:
       - redis
       - postgresql
+      - control_node
     ports:
       - "${ORGANIC_SERVER_PORT:-8091}:${ORGANIC_SERVER_PORT:-8091}"
-    entrypoint: ["opentelemetry-instrument", "--logs_exporter", "otlp", "--traces_exporter", "none", "--metrics_exporter", "otlp", "--service_name", "entry_node", "python", "-u", "/app/validator/entry_node/src/server.py"]
+    volumes:
+      - ${HOME}/.bittensor/wallets:/root/.bittensor/wallets
+    entrypoint: ["opentelemetry-instrument", "--logs_exporter", "otlp", "--traces_exporter", "none", "--metrics_exporter", "otlp", "--service_name", "organic_node", "python", "-u", "/app/validator/organic_node/src/server.py"]
     environment:
       OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
@@ -112,7 +114,7 @@ services:
     env_file:
       - .vali.env
     profiles:
-      - entry_node_profile
+      - organic_node_profile
 
   postgresql:
     container_name: 0_postgresql

dockerfiles/entry_node.dockerfile → dockerfiles/organic_node.dockerfile

@@ -22,22 +22,23 @@ ARG BREAK_CACHE_ARG=0
 RUN pip install --no-cache-dir "git+https://github.com/rayonlabs/[email protected]#egg=fiber[full]"
 ################################################################################
 
-FROM core AS entry_node
+FROM core AS organic_node
 
-WORKDIR /app/validator/entry_node
+WORKDIR /app/validator/organic_node
 
-COPY validator/entry_node/requirements.txt .
+COPY validator/organic_node/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY core /app/core
 COPY validator/utils /app/validator/utils
 COPY validator/models.py /app/validator/models.py
 COPY validator/db /app/validator/db
 
-COPY validator/entry_node/src ./src
-COPY validator/entry_node/pyproject.toml .
+COPY validator/organic_node/src ./src
+COPY validator/common /app/validator/common
+COPY validator/organic_node/pyproject.toml .
 
 
-ENV PYTHONPATH="${PYTHONPATH}:/app/validator/entry_node/src"
+ENV PYTHONPATH="${PYTHONPATH}:/app/validator/organic_node/src"
 
-# CMD ["tail", "-f", "/dev/null"]
+# CMD ["tail", "-f", "/dev/null"]

dockerfiles/query_node.dockerfile → dockerfiles/synthetic_node.dockerfile

@@ -22,22 +22,23 @@ ARG BREAK_CACHE_ARG=0
 RUN pip install --no-cache-dir "git+https://github.com/rayonlabs/[email protected]#egg=fiber[full]"
 ################################################################################
 
-FROM core AS query_node
+FROM core AS synthetic_node
 
-WORKDIR /app/validator/query_node
+WORKDIR /app/validator/synthetic_node
 
-COPY validator/query_node/requirements.txt .
+COPY validator/synthetic_node/requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 COPY core /app/core
 COPY validator/utils /app/validator/utils
 COPY validator/models.py /app/validator/models.py
 COPY validator/db /app/validator/db
 
-COPY validator/query_node/src ./src
-COPY validator/query_node/pyproject.toml .
+COPY validator/synthetic_node/src ./src
+COPY validator/common /app/validator/common
+COPY validator/synthetic_node/pyproject.toml .
 
 
-ENV PYTHONPATH="${PYTHONPATH}:/app/validator/query_node/src"
+ENV PYTHONPATH="${PYTHONPATH}:/app/validator/synthetic_node/src"
 
-# CMD ["tail", "-f", "/dev/null"]
+# CMD ["tail", "-f", "/dev/null"]