diff --git a/.DS_Store b/.DS_Store index eeefc0c2..e2d54922 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 4f8c5dcb..3f36483b 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,5 @@ typesense-data/ redis-data/ env.local +.env.local Kafka \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index ccfa6631..352a1586 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -7,7 +7,6 @@ services: dockerfile: packages/ocular-models-server/Dockerfile.models ports: - 8000:8000 - qdrant: image: qdrant/qdrant:latest restart: always @@ -19,9 +18,6 @@ services: - 6333 - 6334 - 6335 - configs: - - source: qdrant_config - target: /qdrant/config/production.yaml volumes: - ./qdrant_data:/qdrant_data @@ -34,27 +30,32 @@ services: - ./redis-data:/data zookeeper: - image: "bitnami/zookeeper:latest" + image: "confluentinc/cp-zookeeper:latest" ports: - "2181:2181" environment: - - ALLOW_ANONYMOUS_LOGIN=yes + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + volumes: + - ./log4j.properties:/opt/bitnami/zookeeper/conf/log4j.properties kafka: - image: "bitnami/kafka:latest" + image: confluentinc/cp-kafka:latest user: root ports: - "9092:9092" + depends_on: + - zookeeper environment: - KAFKA_BROKER_ID=1 - KAFKA_LISTENERS=PLAINTEXT://:9092 - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9092 - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR volumes: - ./Kafka:/bitnami/kafka - depends_on: - - zookeeper + - ./log4j.properties:/etc/kafka/log4j.properties ocular-db: image: postgres:14-alpine @@ -75,7 +76,7 @@ services: environment: - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular - DATABASE_NAME=ocular - command: npm run typeorm migration:run + command: npm run typeorm:local migration:run pgadmin: image: dpage/pgadmin4 diff --git a/docker-compose.local.yml b/docker-compose.local.yml index cb52a36f..670bcdb2 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,5 +1,12 @@ version: "3" services: + ocular-models-server: + container_name: ocular-models-server + build: + context: . + dockerfile: packages/ocular-models-server/Dockerfile.models + ports: + - 8000:8000 qdrant: image: qdrant/qdrant:latest restart: always @@ -11,9 +18,6 @@ services: - 6333 - 6334 - 6335 - configs: - - source: qdrant_config - target: /qdrant/config/production.yaml volumes: - ./qdrant_data:/qdrant_data @@ -25,27 +29,46 @@ services: volumes: - ./redis-data:/data + zookeeper: + image: "confluentinc/cp-zookeeper:latest" + ports: + - "2181:2181" + environment: + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 + volumes: + - ./log4j.properties:/opt/bitnami/zookeeper/conf/log4j.properties + + kafka: + image: confluentinc/cp-kafka:latest + user: root + ports: + - "9092:9092" + - "29092:29092" + depends_on: + - zookeeper + environment: + - KAFKA_BROKER_ID=1 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_LISTENERS=PLAINTEXT://:9092,EXTERNAL://kafka:29092 + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://:9092,EXTERNAL://kafka:29092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + - ALLOW_PLAINTEXT_LISTENER=yes + - KAFKA_LOG4J_LOGGERS=org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR + volumes: + - ./Kafka:/bitnami/kafka + - ./log4j.properties:/etc/kafka/log4j.properties + ocular-db: image: postgres:14-alpine ports: - - "5432:5432" + - "5433:5432" environment: POSTGRES_PASSWORD: ocular POSTGRES_USER: ocular POSTGRES_DB: ocular - ocular-db-migration: - container_name: ocular-db-migration - depends_on: - - ocular-db - build: - context: . - dockerfile: packages/ocular/Dockerfile.local - environment: - - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular - - DATABASE_NAME=ocular - command: npm run typeorm migration:run - ocular-backend: container_name: ocular-backend build: @@ -54,22 +77,24 @@ services: depends_on: ocular-db: condition: service_started - ocular-db-migration: - condition: service_completed_successfully qdrant: condition: service_started - # typesense: - # condition: service_started + ocular-models-server: + condition: service_started + kafka: + condition: service_started ports: - 9000:9000 + command: /bin/sh -c "npm run typeorm:local migration:run && npm run local" environment: - DATABASE_URL=postgresql://ocular:ocular@ocular-db:5432/ocular - DATABASE_NAME=ocular - NODE_ENV=development - QDRANT_DB_URL=http://qdrant:6333 - # - TYPESENSE_HOST=typesense - UI_CORS=http://ocular-ui:9000 - REDIS_URL=redis:6379 + - KAFKA_URL=kafka:29092 + - OCULAR_MODELS_SERVER_URL=http://ocular-models-server:8000 ocular-ui: container_name: ocular-ui @@ -80,8 +105,3 @@ services: - 3001:3001 depends_on: - ocular-backend - -configs: - qdrant_config: - content: | - log_level: INFO diff --git a/env.dev.example b/env.dev similarity index 59% rename from env.dev.example rename to env.dev index b032a4ab..6eeb429f 100644 --- a/env.dev.example +++ b/env.dev @@ -1,4 +1,3 @@ -# Database URL + REDIS + QDRANT running Docker NODE_ENV=development DATABASE_URL=postgresql://ocular:ocular@localhost:5433/ocular DATABASE_NAME=ocular @@ -6,22 +5,14 @@ QDRANT_DB_URL=http://localhost:6333 UI_CORS=http://localhost:9000 REDIS_URL=redis://localhost:6379 KAFKA_URL=localhost:9092 - - -# Required Azure Open AI credentials -AZURE_OPEN_AI_KEY= -AZURE_OPEN_AI_ENDPOINT= -AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME= -AZURE_OPEN_AI_EMBEDDING_MODEL= -AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME= -AZURE_OPEN_AI_CHAT_MODEL= +USE_LOCAL_EMBEDDER=true # Use local embedder instead of Azure or OpenAI +OCULAR_MODELS_SERVER_URL=http://localhost:8000 # Required Open AI credentials OPEN_AI_KEY= -OPEN_AI_EMBEDDING_MODEL= OPEN_AI_CHAT_MODEL= -# Add these if you want to enable index data from apps +# Add these if you want to use OAuth to index data from apps # Google (Gmail + Google Drive) GOOGLE_CLIENT_ID= diff --git a/env.local b/env.local new file mode 100644 index 00000000..8d181a49 --- /dev/null +++ b/env.local @@ -0,0 +1,11 @@ +# Required Open AI Keys +OPEN_AI_KEY= +OPEN_AI_CHAT_MODEL= + +# OAUTH Credientials - Add these if you want to enable OAUTH for indexing apps +# Apps Installed +GOOGLE_CLIENT_ID= +GOOGLE_CLIENT_SECRET= + +ASANA_CLIENT_ID= +ASANA_CLIENT_SECRET= diff --git a/env.local.example b/env.local.example deleted file mode 100644 index 6a541105..00000000 --- a/env.local.example +++ /dev/null @@ -1,22 +0,0 @@ - -# Required Azure Open AI credentials -AZURE_OPEN_AI_KEY= -AZURE_OPEN_AI_ENDPOINT= -AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME= -AZURE_OPEN_AI_EMBEDDING_MODEL= -AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME= -AZURE_OPEN_AI_CHAT_MODEL= - -# Apps Installed -GOOGLE_CLIENT_ID= -GOOGLE_CLIENT_SECRET= - -# Asana -ASANA_CLIENT_ID= -ASANA_CLIENT_SECRET= - -# GitHub -GITHUB_APP_ID= -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= -GITHUB_PRIVATE_KEY_PATH= \ No newline at end of file diff --git a/log4j.properties b/log4j.properties new file mode 100644 index 00000000..6698fa0b --- /dev/null +++ b/log4j.properties @@ -0,0 +1,23 @@ + +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + +log4j.logger.kafka=ERROR +log4j.logger.kafka.network.RequestChannel$=WARN +log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG +log4j.logger.kafka.request.logger=WARN +log4j.logger.kafka.controller=ERROR +log4j.logger.kafka.log.LogCleaner=INFO +log4j.logger.state.change.logger=ERROR +log4j.logger.kafka.authorizer.logger=WARN +log4j.logger.org.apache.zookeeper=ERROR +log4j.logger.org.apache.kafka=ERROR +log4j.logger.kafka.cluster=ERROR +log4j.logger.kafka.coordinator=ERROR +log4j.logger.kafka.log=ERROR +log4j.logger.kafka.server=ERROR +log4j.logger.kafka.zookeeper=ERROR diff --git a/packages/ocular-models-server/server.py b/packages/ocular-models-server/server.py index fc50bb4b..d34d2d4c 100644 --- a/packages/ocular-models-server/server.py +++ b/packages/ocular-models-server/server.py @@ -17,7 +17,6 @@ model.eval() def embed(docs: list[str]) -> list[list[float]]: - print(docs) # tokenize tokens = tokenizer( docs, padding=True, max_length=512, truncation=True, return_tensors="pt" diff --git a/packages/ocular-telemetry/src/telemeter.js b/packages/ocular-telemetry/src/telemeter.js index 9f94a332..fc780011 100644 --- a/packages/ocular-telemetry/src/telemeter.js +++ b/packages/ocular-telemetry/src/telemeter.js @@ -51,7 +51,7 @@ class Telemeter { let enabled = this.store_.getConfig(`telemetry.enabled`) if (enabled === undefined || enabled === null) { if (!isCI()) { - showAnalyticsNotification() + // showAnalyticsNotification() } enabled = true this.store_.setConfig(`telemetry.enabled`, enabled) diff --git a/packages/ocular-telemetry/src/util/show-notification.js b/packages/ocular-telemetry/src/util/show-notification.js index 17a434d1..64347b2f 100644 --- a/packages/ocular-telemetry/src/util/show-notification.js +++ b/packages/ocular-telemetry/src/util/show-notification.js @@ -8,7 +8,7 @@ const defaultConfig = { const defaultMessage = `Ocular collects anonymous usage analytics\n` + - `to help improve Medusa for all users.\n` + + `to help improve Ocular for all users.\n` + `\n` + `If you'd like to opt-out, you can use \`ocular telemetry --disable\`\n` diff --git a/packages/ocular/.env.local.docker b/packages/ocular/.env.local.docker deleted file mode 100644 index 056545f1..00000000 --- a/packages/ocular/.env.local.docker +++ /dev/null @@ -1,20 +0,0 @@ -# Azure Open AI -AZURE_OPEN_AI_KEY= -AZURE_OPEN_AI_ENDPOINT= -AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME= -AZURE_OPEN_AI_EMBEDDING_MODEL= -AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME= -AZURE_OPEN_AI_CHAT_MODEL= - - -# App Keys -ASANA_CLIENT_ID= -ASANA_CLIENT_SECRET= - -GOOGLE_CLIENT_ID= -GOOGLE_CLIENT_SECRET= - -GITHUB_CLIENT_ID= -GITHUB_CLIENT_SECRET= -GITHUB_APP_ID= -GITHUB_PRIVATE_KEY_PATH= \ No newline at end of file diff --git a/packages/ocular/.env.local.example b/packages/ocular/.env.local.example deleted file mode 100644 index 23c25227..00000000 --- a/packages/ocular/.env.local.example +++ /dev/null @@ -1,20 +0,0 @@ -JWT_SECRET=your_jwt_secret_here -COOKIE_SECRET=your_cookie_secret_here -DATABASE_URL=your_database_url_here -DATABASE_NAME=your_database_name_here -REDIS_URL=your_redis_url_here -UI_CORS=your_ui_cors_here -AZURE_OPENAI_API_KEY=your_azure_openai_api_key_here -AZURE_OPEN_AI_SERVICE_NAME=your_azure_open_ai_service_name_here -AZURE_OPENAI_API_VERSION=your_azure_openai_api_version_here -AZURE_OPENAI_DEPLOYMENT_NAME=your_azure_openai_deployment_name_here -GOOGLE_CLIENT_ID=your_google_client_id_here -GOOGLE_CLIENT_SECRET=your_google_client_secret_here -AZURE_OPEN_AI_KEY=your_azure_open_ai_key_here -AZURE_OPEN_AI_ENDPOINT=your_azure_open_ai_endpoint_here -AZURE_OPEN_AI_EMBEDDER_DEPLOYMENT_NAME=your_azure_open_ai_embedder_deployment_name_here -AZURE_OPEN_AI_EMBEDDING_MODEL=your_azure_open_ai_embedding_model_here -AZURE_OPEN_AI_CHAT_DEPLOYMENT_NAME=your_azure_open_ai_chat_deployment_name_here -AZURE_OPEN_AI_CHAT_MODEL=your_azure_open_ai_chat_model_here -QDRANT_DB_URL=your_qdrant_db_url_here -TYPESENSE_HOST=your_typesense_host_here \ No newline at end of file diff --git a/packages/ocular/Dockerfile.dev b/packages/ocular/Dockerfile.dev index b2ef8292..2167fc7b 100644 --- a/packages/ocular/Dockerfile.dev +++ b/packages/ocular/Dockerfile.dev @@ -4,7 +4,6 @@ FROM node:18-alpine RUN apk update && \ apk add chromium - WORKDIR /usr/src/app RUN npm cache clean --force @@ -20,7 +19,7 @@ COPY package-lock.json ./ # Copy the docs package.json COPY packages/ocular/package.json ./packages/ocular/package.json -# COPY packages/ocular/env.local ./packages/ocular/.env.local +COPY env.local ./packages/ocular/.env.local # Copy turbo.json COPY turbo.json ./ @@ -31,12 +30,4 @@ COPY . . RUN npm install --verbose RUN turbo build -EXPOSE 8080 - -WORKDIR /usr/src/app/packages/ocular - -RUN apk add --no-cache bash curl && curl -1sLf \ - 'https://dl.cloudsmith.io/public/infisical/infisical-cli/setup.alpine.sh' | bash \ - && apk add infisical - -CMD ["sh", "-c", "infisical run --env=dev -- npm run start"] \ No newline at end of file +WORKDIR /usr/src/app/packages/ocular \ No newline at end of file diff --git a/packages/ocular/Dockerfile.local b/packages/ocular/Dockerfile.local index 687b31aa..35562efe 100644 --- a/packages/ocular/Dockerfile.local +++ b/packages/ocular/Dockerfile.local @@ -1,6 +1,10 @@ FROM node:18-alpine + +RUN apk update && \ + apk add chromium + WORKDIR /usr/src/app RUN npm cache clean --force @@ -27,8 +31,4 @@ COPY . . RUN npm install --verbose RUN turbo build -EXPOSE 8080 - WORKDIR /usr/src/app/packages/ocular - -CMD ["NODE_ENV", "=", "development", "npm", "run", "start"] \ No newline at end of file diff --git a/packages/ocular/core-config.js b/packages/ocular/core-config-dev.js similarity index 98% rename from packages/ocular/core-config.js rename to packages/ocular/core-config-dev.js index 7f3451a1..830f0fd7 100644 --- a/packages/ocular/core-config.js +++ b/packages/ocular/core-config-dev.js @@ -3,6 +3,8 @@ const dotenv = require("dotenv"); let ENV_FILE_NAME = ""; switch (process.env.NODE_ENV) { + case "local": + ENV_FILE_NAME = ".env.local"; case "production": ENV_FILE_NAME = ".env.production"; break; @@ -16,7 +18,7 @@ switch (process.env.NODE_ENV) { ENV_FILE_NAME = ".env.dev"; break; default: - ENV_FILE_NAME = ".env.dev"; + ENV_FILE_NAME = ".env.local"; break; } diff --git a/packages/ocular/core-config-local.js b/packages/ocular/core-config-local.js new file mode 100644 index 00000000..91b446b9 --- /dev/null +++ b/packages/ocular/core-config-local.js @@ -0,0 +1,114 @@ +const { PluginNameDefinitions } = require("@ocular/types"); +const dotenv = require("dotenv"); + +let ENV_FILE_NAME = ""; +switch (process.env.NODE_ENV) { + case "local": + ENV_FILE_NAME = ".env.local"; + break; + case "production": + ENV_FILE_NAME = ".env.production"; + break; + case "staging": + ENV_FILE_NAME = ".env.staging"; + break; + case "test": + ENV_FILE_NAME = ".env.test"; + break; + case "development": + ENV_FILE_NAME = ".env.dev"; + break; + default: + ENV_FILE_NAME = ".env.local"; + break; +} + +try { + dotenv.config({ path: process.cwd() + "/" + ENV_FILE_NAME }); +} catch (e) {} + +// UI_CORS is the URL of the UI that is allowed to access the API +const UI_CORS = process.env.ADMIN_CORS || "http://localhost:3001"; + +/** @type {import('./src/types/config-module').ConfigModule} */ +module.exports = { + projectConfig: { + jwtSecret: process.env.JWT_SECRET, + cookieSecret: process.env.COOKIE_SECRET, + database_url: process.env.DATABASE_URL, + database_database: process.env.DATABASE_NAME, + database_type: "postgres", + redis_url: process.env.REDIS_URL, + kafka_url: process.env.KAFKA_URL, + ui_cors: UI_CORS, + }, + apps: [ + { + resolve: `asana`, + options: { + client_id: process.env.ASANA_CLIENT_ID, + client_secret: process.env.ASANA_CLIENT_SECRET, + scope: "openid email profile", + redirect_uri: `${UI_CORS}/dashboard/marketplace/asana`, + rate_limiter_opts: { + requests: 1500, // Number of Requests + interval: 60, // Interval in Seconds + }, + }, + }, + { + resolve: `google-drive`, + options: { + client_id: process.env.GOOGLE_CLIENT_ID, + client_secret: process.env.GOOGLE_CLIENT_SECRET, + redirect_uri: `${UI_CORS}/dashboard/marketplace/google-drive`, + rate_limiter_opts: { + requests: 60, // Number of Requests + interval: 60, // Interval in Seconds + }, + }, + }, + { + resolve: `webConnector`, + options: { + client_id: "FAKE_ID", + client_secret: "FAKE_SECRET", + redirect_uri: `${UI_CORS}/dashboard/marketplace/webConnector`, + }, + }, + ], + plugins: [ + { + resolve: `document-processor`, + options: { + max_chunk_length: 500, + sentence_search_limit: 100, + chunk_over_lap: 0, + }, + }, + { + resolve: PluginNameDefinitions.OPENAI, + options: { + open_ai_key: process.env.OPEN_AI_KEY, + chat_model: process.env.OPEN_AI_CHAT_MODEL, + rate_limiter_opts: { + requests: 1000000, // Number of Tokens + interval: 60, // Interval in Seconds + }, + }, + }, + { + resolve: `qdrant-vector-search-service`, + options: { + quadrant_db_url: process.env.QDRANT_DB_URL || "http://localhost:6333", + embedding_size: 768, + }, + }, + { + resolve: `embedder`, + options: { + models_server_url: process.env.OCULAR_MODELS_SERVER_URL, + }, + }, + ], +}; \ No newline at end of file diff --git a/packages/ocular/package.json b/packages/ocular/package.json index 643f6531..c0cc608f 100644 --- a/packages/ocular/package.json +++ b/packages/ocular/package.json @@ -22,11 +22,13 @@ "watch": "tsc --build --watch", "prepublishOnly": "cross-env NODE_ENV=production tsc --build", "build": "rimraf dist && tsc --build", - "dev": "npm run build && nodemon dist/ocular/src/main.js", + "dev": "cross-env CONFIG_FILE=core-config-local.js NODE_ENV=development npm run start", + "local": "cross-env CONFIG_FILE=core-config-local.js NODE_ENV=local npm run start", "start": "npm run build && nodemon dist/ocular/src/main.js", "clean": "rimraf dist node_modules", "test": "jest src --detectOpenHandles", - "typeorm": "npm run build && typeorm-ts-node-esm -d dist/ocular/src/data-source.js" + "typeorm:dev": "npm run build && cross-env CONFIG_FILE=core-config-dev.js typeorm-ts-node-esm -d dist/ocular/src/data-source.js", + "typeorm:local": "npm run build && cross-env CONFIG_FILE=core-config-local.js typeorm-ts-node-esm -d dist/ocular/src/data-source.js" }, "dependencies": { "@azure/search-documents": "^12.0.0", diff --git a/packages/ocular/src/data-source.ts b/packages/ocular/src/data-source.ts index 962b09a0..347e0c11 100644 --- a/packages/ocular/src/data-source.ts +++ b/packages/ocular/src/data-source.ts @@ -15,7 +15,10 @@ const fg = require("fast-glob"); const rootDirectory = path.resolve(`.`); const getMigrations = (directory) => { - const { configModule, error } = getConfigFile(directory, `core-config`); + const { configModule, error } = getConfigFile( + directory, + process.env.CONFIG_FILE + ); const migrationDirs = []; const corePackageMigrations = path.resolve( diff --git a/packages/ocular/src/loaders/config.ts b/packages/ocular/src/loaders/config.ts index d0994e24..e74641cd 100644 --- a/packages/ocular/src/loaders/config.ts +++ b/packages/ocular/src/loaders/config.ts @@ -1,33 +1,41 @@ -import getConfigFile from "../utils/get-config-file" -import {ConfigModule} from "../types/config-module" +import getConfigFile from "../utils/get-config-file"; +import { ConfigModule } from "../types/config-module"; -const isProduction = ["production", "prod"].includes(process.env.NODE_ENV || "") +const isProduction = ["production", "prod"].includes( + process.env.NODE_ENV || "" +); const errorHandler = isProduction ? (msg: string): never => { - throw new Error(msg) + throw new Error(msg); } - : console.log - + : console.log; export default (rootDirectory: string): ConfigModule => { + const config_file = process.env.CONFIG_FILE; + if (!config_file) { + throw new Error( + `[core-config] ⚠️ CONFIG_FILE not found. Fallback to default config file.` + ); + } + const { configModule, error } = getConfigFile( rootDirectory, - `core-config.js` - ) + config_file + ); if (error) { - console.log("Error Loading Config",error) + console.log("Error Loading Config", error); } if (!configModule?.projectConfig?.redis_url) { console.log( `[core-config] ⚠️ redis_url not found. A fake redis instance will be used.` - ) + ); } const jwt_secret = - configModule?.projectConfig?.jwt_secret ?? process.env.JWT_SECRET + configModule?.projectConfig?.jwt_secret ?? process.env.JWT_SECRET; if (!jwt_secret) { errorHandler( `[core-config] ⚠️ jwt_secret not found.${ @@ -35,11 +43,11 @@ export default (rootDirectory: string): ConfigModule => { ? "" : " fallback to either cookie_secret or default 'supersecret'." }` - ) + ); } const cookie_secret = - configModule?.projectConfig?.cookie_secret ?? process.env.COOKIE_SECRET + configModule?.projectConfig?.cookie_secret ?? process.env.COOKIE_SECRET; if (!cookie_secret) { errorHandler( `[core-config] ⚠️ cookie_secret not found.${ @@ -47,7 +55,7 @@ export default (rootDirectory: string): ConfigModule => { ? "" : " fallback to either cookie_secret or default 'supersecret'." }` - ) + ); } return { @@ -59,5 +67,5 @@ export default (rootDirectory: string): ConfigModule => { // modules: configModule?.modules ?? {}, apps: configModule?.apps ?? [], plugins: configModule?.plugins ?? [], - } -} + }; +}; diff --git a/packages/ocular/src/loaders/kafka.ts b/packages/ocular/src/loaders/kafka.ts index 23c9477b..767332e3 100644 --- a/packages/ocular/src/loaders/kafka.ts +++ b/packages/ocular/src/loaders/kafka.ts @@ -17,25 +17,24 @@ async function kafkaLoader({ configModule, logger, }: Options): Promise { + console.log("Kafka Loader", configModule.projectConfig.kafka_url); if (configModule.projectConfig.kafka_url) { - const kafkaClient = new Kafka({ - clientId: "ocular", - brokers: [configModule.projectConfig.kafka_url], - logLevel: logLevel.ERROR, - }); - try { + const kafkaClient = new Kafka({ + clientId: "ocular", + brokers: [configModule.projectConfig.kafka_url], + logLevel: logLevel.ERROR, + }); await kafkaClient.admin().connect(); logger?.info(`Connection to Kafka established`); + container.register({ + kafkaClient: asValue(kafkaClient), + }); } catch (err) { throw new Error( `An error occurred while connecting to Kafka:${EOL} ${err}` ); } - - container.register({ - kafkaClient: asValue(kafkaClient), - }); } else { throw new Error( `No Kafka url was provided - using Ocular without a proper Kafka instance is allowed` diff --git a/packages/ocular/src/loaders/logger.ts b/packages/ocular/src/loaders/logger.ts index 46cdb818..a2488a79 100644 --- a/packages/ocular/src/loaders/logger.ts +++ b/packages/ocular/src/loaders/logger.ts @@ -1,18 +1,17 @@ -import stackTrace from "stack-trace" -import { ulid } from "ulid" -import winston from "winston" -import ora from "ora" -import * as Transport from "winston-transport" - +import stackTrace from "stack-trace"; +import { ulid } from "ulid"; +import winston from "winston"; +import ora from "ora"; +import * as Transport from "winston-transport"; // Panic Handler export type PanicData = { - id: string + id: string; context: { - rootPath: string - path: string - } -} + rootPath: string; + path: string; + }; +}; export enum PanicId { InvalidProjectName = "10000", @@ -21,37 +20,37 @@ export enum PanicId { } export const panicHandler = (panicData: PanicData = {} as PanicData) => { - const { id, context } = panicData + const { id, context } = panicData; switch (id) { case "10000": return { message: `Looks like you provided a URL as your project name. Please provide a project name instead.`, - } + }; case "10002": return { message: `Could not create project because ${context.path} is not a valid path.`, - } + }; case "10003": return { message: `Directory ${context.rootPath} is already a Node project.`, - } + }; default: return { message: "Unknown error", - } + }; } -} +}; // Logger -const LOG_LEVEL = process.env.LOG_LEVEL || "silly" -const LOG_FILE = process.env.LOG_FILE || "" -const NODE_ENV = process.env.NODE_ENV || "development" -const IS_DEV = NODE_ENV.startsWith("dev") +const LOG_LEVEL = process.env.LOG_LEVEL || "silly"; +const LOG_FILE = process.env.LOG_FILE || ""; +const NODE_ENV = process.env.NODE_ENV || "development"; +const IS_DEV = NODE_ENV.startsWith("dev") || NODE_ENV.startsWith("local"); -let transports: Transport[] = [] +let transports: Transport[] = []; if (!IS_DEV) { - transports.push(new winston.transports.Console()) + transports.push(new winston.transports.Console()); } else { transports.push( new winston.transports.Console({ @@ -60,15 +59,15 @@ if (!IS_DEV) { winston.format.splat() ), }) - ) + ); } if (LOG_FILE) { transports.push( new winston.transports.File({ - filename: LOG_FILE + filename: LOG_FILE, }) - ) + ); } const loggerInstance = winston.createLogger({ @@ -83,30 +82,30 @@ const loggerInstance = winston.createLogger({ winston.format.json() ), transports, -}) +}); export class Reporter { - protected activities_: Record - protected loggerInstance_: winston.Logger - protected ora_: typeof ora + protected activities_: Record; + protected loggerInstance_: winston.Logger; + protected ora_: typeof ora; constructor({ logger, activityLogger }) { - this.activities_ = {} - this.loggerInstance_ = logger - this.ora_ = activityLogger + this.activities_ = {}; + this.loggerInstance_ = logger; + this.ora_ = activityLogger; } panic = (data) => { - const parsedPanic = panicHandler(data) + const parsedPanic = panicHandler(data); this.loggerInstance_.log({ level: "error", details: data, message: parsedPanic.message, - }) + }); - process.exit(1) - } + process.exit(1); + }; /** * Determines if the logger should log at a given level. @@ -114,26 +113,26 @@ export class Reporter { * @return {boolean} whether we should log */ shouldLog = (level) => { - level = this.loggerInstance_.levels[level] - const logLevel = this.loggerInstance_.levels[this.loggerInstance_.level] - return level <= logLevel - } + level = this.loggerInstance_.levels[level]; + const logLevel = this.loggerInstance_.levels[this.loggerInstance_.level]; + return level <= logLevel; + }; /** * Sets the log level of the logger. * @param {string} level - the level to set the logger to */ setLogLevel = (level) => { - this.loggerInstance_.level = level - } + this.loggerInstance_.level = level; + }; /** * Resets the logger to the value specified by the LOG_LEVEL env var. If no * LOG_LEVEL is set it defaults to "silly". */ unsetLogLevel = () => { - this.loggerInstance_.level = LOG_LEVEL - } + this.loggerInstance_.level = LOG_LEVEL; + }; /** * Begin an activity. In development an activity is displayed as a spinner; @@ -144,32 +143,32 @@ export class Reporter { * further operations on the activity such as success, failure, progress. */ activity = (message, config = {}) => { - const id = ulid() + const id = ulid(); if (IS_DEV && this.shouldLog("info")) { - const activity = this.ora_(message).start() + const activity = this.ora_(message).start(); this.activities_[id] = { activity, config, start: Date.now(), - } + }; - return id + return id; } else { this.activities_[id] = { start: Date.now(), config, - } + }; this.loggerInstance_.log({ activity_id: id, level: "info", config, message, - }) + }); - return id + return id; } - } + }; /** * Reports progress on an activity. In development this will update the @@ -182,20 +181,20 @@ export class Reporter { const toLog = { level: "info", message, - } + }; if (typeof activityId === "string" && this.activities_[activityId]) { - const activity = this.activities_[activityId] + const activity = this.activities_[activityId]; if (activity.activity) { - activity.text = message + activity.text = message; } else { - toLog["activity_id"] = activityId - this.loggerInstance_.log(toLog) + toLog["activity_id"] = activityId; + this.loggerInstance_.log(toLog); } } else { - this.loggerInstance_.log(toLog) + this.loggerInstance_.log(toLog); } - } + }; /** * Logs an error. If an error object is provided the stack trace for the error @@ -205,28 +204,28 @@ export class Reporter { * @param {Error?} error - an error object to log message with */ error = (messageOrError, error = null) => { - let message = messageOrError + let message = messageOrError; if (typeof messageOrError === "object") { - message = messageOrError.message - error = messageOrError + message = messageOrError.message; + error = messageOrError; } const toLog = { level: "error", message, - } + }; if (error) { - toLog["stack"] = stackTrace.parse(error) + toLog["stack"] = stackTrace.parse(error); } - this.loggerInstance_.log(toLog) + this.loggerInstance_.log(toLog); // Give stack traces and details in dev if (error && IS_DEV) { - console.log(error) + console.log(error); } - } + }; /** * Reports failure of an activity. In development the activity will be udpated @@ -237,35 +236,35 @@ export class Reporter { * @returns {object} data about the activity */ failure = (activityId, message) => { - const time = Date.now() + const time = Date.now(); const toLog = { level: "error", message, - } + }; if (typeof activityId === "string" && this.activities_[activityId]) { - const activity = this.activities_[activityId] + const activity = this.activities_[activityId]; if (activity.activity) { - activity.activity.fail(`${message} – ${time - activity.start}`) + activity.activity.fail(`${message} – ${time - activity.start}`); } else { - toLog["duration"] = time - activity.start - toLog["activity_id"] = activityId - this.loggerInstance_.log(toLog) + toLog["duration"] = time - activity.start; + toLog["activity_id"] = activityId; + this.loggerInstance_.log(toLog); } } else { - this.loggerInstance_.log(toLog) + this.loggerInstance_.log(toLog); } if (this.activities_[activityId]) { - const activity = this.activities_[activityId] + const activity = this.activities_[activityId]; return { ...activity, duration: time - activity.start, - } + }; } - return null - } + return null; + }; /** * Reports success of an activity. In development the activity will be udpated @@ -276,35 +275,35 @@ export class Reporter { * @returns {Record} data about the activity */ success = (activityId, message) => { - const time = Date.now() + const time = Date.now(); const toLog = { level: "info", message, - } + }; if (typeof activityId === "string" && this.activities_[activityId]) { - const activity = this.activities_[activityId] + const activity = this.activities_[activityId]; if (activity.activity) { - activity.activity.succeed(`${message} – ${time - activity.start}ms`) + activity.activity.succeed(`${message} – ${time - activity.start}ms`); } else { - toLog["duration"] = time - activity.start - toLog["activity_id"] = activityId - this.loggerInstance_.log(toLog) + toLog["duration"] = time - activity.start; + toLog["activity_id"] = activityId; + this.loggerInstance_.log(toLog); } } else { - this.loggerInstance_.log(toLog) + this.loggerInstance_.log(toLog); } if (this.activities_[activityId]) { - const activity = this.activities_[activityId] + const activity = this.activities_[activityId]; return { ...activity, duration: time - activity.start, - } + }; } - return null - } + return null; + }; /** * Logs a message at the info level. @@ -314,8 +313,8 @@ export class Reporter { this.loggerInstance_.log({ level: "debug", message, - }) - } + }); + }; /** * Logs a message at the info level. @@ -325,8 +324,8 @@ export class Reporter { this.loggerInstance_.log({ level: "info", message, - }) - } + }); + }; /** * Logs a message at the warn level. @@ -336,21 +335,21 @@ export class Reporter { this.loggerInstance_.warn({ level: "warn", message, - }) - } + }); + }; /** * A wrapper around winston's log method. */ log = (...args) => { // @ts-ignore - this.loggerInstance_.log(...args) - } + this.loggerInstance_.log(...args); + }; } const logger = new Reporter({ logger: loggerInstance, activityLogger: ora, -}) +}); -export default logger \ No newline at end of file +export default logger; diff --git a/packages/ocular/src/services/queue.ts b/packages/ocular/src/services/queue.ts index 73f969a0..3a770a56 100644 --- a/packages/ocular/src/services/queue.ts +++ b/packages/ocular/src/services/queue.ts @@ -127,43 +127,49 @@ export default class QueueService extends AbstractQueueService { consumer: Consumer, context: ConsumerContext ): Promise { - if (typeof consumer !== `function`) { - throw new Error("Subscriber must be a function"); - } - const kafkaConsumer = this.kafkaClient_.consumer({ - groupId: context.groupId, - sessionTimeout: 60000, // 60 seconds - }); - kafkaConsumer.connect(); - kafkaConsumer.subscribe({ topic: topicName, fromBeginning: false }); - kafkaConsumer.run({ - eachBatchAutoResolve: true, - eachBatch: async ({ batch, heartbeat }) => { - try { - const docs: IndexableDocument[] = batch.messages.map((message) => { - return JSON.parse(message.value.toString()) as IndexableDocument; - }); - this.logger_.info( - `eachBatch: Batch Received ${docs.length} messages` - ); - consumer(docs, topic).catch((error) => { + try { + if (typeof consumer !== `function`) { + throw new Error("Subscriber must be a function"); + } + const kafkaConsumer = this.kafkaClient_.consumer({ + groupId: context.groupId, + sessionTimeout: 60000, // 60 seconds + }); + kafkaConsumer.connect(); + kafkaConsumer.subscribe({ topic: topicName, fromBeginning: false }); + kafkaConsumer.run({ + eachBatchAutoResolve: true, + eachBatch: async ({ batch, heartbeat }) => { + try { + const docs: IndexableDocument[] = batch.messages.map((message) => { + return JSON.parse(message.value.toString()) as IndexableDocument; + }); + this.logger_.info( + `eachBatch: Batch Received ${docs.length} messages` + ); + consumer(docs, topic).catch((error) => { + this.logger_.error(`Error processing message: ${error.message}`); + }); + this.logger_.info( + `eachBatch: Batch Processing ${docs.length} done` + ); + await heartbeat(); + } catch (error) { this.logger_.error(`Error processing message: ${error.message}`); - }); - this.logger_.info(`eachBatch: Batch Processing ${docs.length} done`); - await heartbeat(); - } catch (error) { - this.logger_.error(`Error processing message: ${error.message}`); - } - }, - }); + } + }, + }); - const randId = ulid(); - const topic = topicName.toString(); + const randId = ulid(); + const topic = topicName.toString(); - this.storeConsumers({ - topicName, - consumerId: `${topic}-${randId}`, - consumer: kafkaConsumer, - }); + this.storeConsumers({ + topicName, + consumerId: `${topic}-${randId}`, + consumer: kafkaConsumer, + }); + } catch (error) { + this.logger_.error(`Error subscribing to topic: ${error.message}`); + } } } diff --git a/packages/ocular/src/utils/handle-postgres-database-error.ts b/packages/ocular/src/utils/handle-postgres-database-error.ts index ca27a544..8945ed9e 100644 --- a/packages/ocular/src/utils/handle-postgres-database-error.ts +++ b/packages/ocular/src/utils/handle-postgres-database-error.ts @@ -1,4 +1,4 @@ -import { EOL } from "os" +import { EOL } from "os"; export const DatabaseErrorCode = { databaseDoesNotExist: "3D000", @@ -6,13 +6,14 @@ export const DatabaseErrorCode = { wrongCredentials: "28000", notFound: "ENOTFOUND", migrationMissing: "42P01", -} +}; export function handlePostgresDatabaseError(err: any): never { + console.log(err); if (DatabaseErrorCode.databaseDoesNotExist === err.code) { throw new Error( `The specified PostgreSQL database does not exist. Please create it and try again.${EOL}${err.message}` - ) + ); } if (DatabaseErrorCode.connectionFailure === err.code) { @@ -24,26 +25,26 @@ export function handlePostgresDatabaseError(err: any): never { "postgres://[username]:[password]@[host]:[post]/[db_name]" - If there is no password, you can omit it from the connection string ${EOL} ${err.message}` - ) + ); } if (DatabaseErrorCode.wrongCredentials === err.code) { throw new Error( `The specified credentials does not exists for the specified PostgreSQL database.${EOL}${err.message}` - ) + ); } if (DatabaseErrorCode.notFound === err.code) { throw new Error( `The specified connection string for your PostgreSQL database might have illegal characters. Please check that it only contains allowed characters [a-zA-Z0-9]${EOL}${err.message}` - ) + ); } if (DatabaseErrorCode.migrationMissing === err.code) { throw new Error( `Migrations missing. Please run 'npm typeorm:migrate' and try again.` - ) + ); } - throw err + throw err; } diff --git a/packages/plugins/open-ai/src/services/open-ai.ts b/packages/plugins/open-ai/src/services/open-ai.ts index 9be8856b..1901dacf 100644 --- a/packages/plugins/open-ai/src/services/open-ai.ts +++ b/packages/plugins/open-ai/src/services/open-ai.ts @@ -29,7 +29,6 @@ export default class OpenAIService extends AbstractLLMService { this.requestQueue_ = this.rateLimiterService_.getRequestQueue( PluginNameDefinitions.OPENAI ); - // Models this.embeddingModel_ = options.embedding_model; this.chatModel_ = options.chat_model;