diff --git a/.changeset/thirty-beds-sniff.md b/.changeset/thirty-beds-sniff.md new file mode 100644 index 000000000..41525e4e0 --- /dev/null +++ b/.changeset/thirty-beds-sniff.md @@ -0,0 +1,5 @@ +--- +"create-llama": minor +--- + +Add Llama3 and Phi3 support using Ollama diff --git a/create-app.ts b/create-app.ts index 5f6fd1366..d8c5ae3ad 100644 --- a/create-app.ts +++ b/create-app.ts @@ -30,10 +30,8 @@ export async function createApp({ appPath, packageManager, frontend, - openAiKey, + modelConfig, llamaCloudKey, - model, - embeddingModel, communityProjectConfig, llamapack, vectorDb, @@ -77,10 +75,8 @@ export async function createApp({ ui, packageManager, isOnline, - openAiKey, + modelConfig, llamaCloudKey, - model, - embeddingModel, communityProjectConfig, llamapack, vectorDb, diff --git a/helpers/env-variables.ts b/helpers/env-variables.ts index fec6e8014..9cd3f5ad0 100644 --- a/helpers/env-variables.ts +++ b/helpers/env-variables.ts @@ -1,6 +1,7 @@ import fs from "fs/promises"; import path from "path"; import { + ModelConfig, TemplateDataSource, TemplateFramework, TemplateVectorDB, @@ -28,7 +29,10 @@ const renderEnvVar = (envVars: EnvVar[]): string => { ); }; -const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => { +const getVectorDBEnvs = (vectorDb?: TemplateVectorDB): EnvVar[] => { + if (!vectorDb) { + return []; + } switch (vectorDb) { case "mongo": return [ @@ -130,84 +134,75 @@ const getVectorDBEnvs = (vectorDb: TemplateVectorDB) => { } }; -export const createBackendEnvFile = async ( - root: string, - opts: { - openAiKey?: string; - llamaCloudKey?: string; - vectorDb?: TemplateVectorDB; - model?: string; - embeddingModel?: string; - framework?: TemplateFramework; - dataSources?: TemplateDataSource[]; - port?: number; - }, -) => { - // Init env values - const envFileName = ".env"; - const defaultEnvs = [ - { - render: true, - name: "MODEL", - description: "The name of LLM model to use.", - value: opts.model, - }, +const getModelEnvs = (modelConfig: ModelConfig): EnvVar[] => { + return [ { - render: true, - name: "OPENAI_API_KEY", - description: "The OpenAI API key to use.", - value: opts.openAiKey, + name: "MODEL_PROVIDER", + description: "The provider for the AI models to use.", + value: modelConfig.provider, }, { - name: "LLAMA_CLOUD_API_KEY", - description: `The Llama Cloud API key.`, - value: opts.llamaCloudKey, + name: "MODEL", + description: "The name of LLM model to use.", + value: modelConfig.model, }, { name: "EMBEDDING_MODEL", description: "Name of the embedding model to use.", - value: opts.embeddingModel, + value: modelConfig.embeddingModel, }, { name: "EMBEDDING_DIM", description: "Dimension of the embedding model to use.", - value: 1536, + value: modelConfig.dimensions.toString(), }, - // Add vector database environment variables - ...(opts.vectorDb ? getVectorDBEnvs(opts.vectorDb) : []), + ...(modelConfig.provider === "openai" + ? [ + { + name: "OPENAI_API_KEY", + description: "The OpenAI API key to use.", + value: modelConfig.apiKey, + }, + ] + : []), ]; - let envVars: EnvVar[] = []; - if (opts.framework === "fastapi") { - envVars = [ - ...defaultEnvs, - ...[ - { - name: "APP_HOST", - description: "The address to start the backend app.", - value: "0.0.0.0", - }, - { - name: "APP_PORT", - description: "The port to start the backend app.", - value: opts.port?.toString() || "8000", - }, - { - name: "LLM_TEMPERATURE", - description: "Temperature for sampling from the model.", - }, - { - name: "LLM_MAX_TOKENS", - description: "Maximum number of tokens to generate.", - }, - { - name: "TOP_K", - description: - "The number of similar embeddings to return when retrieving documents.", - value: "3", - }, - { - name: "SYSTEM_PROMPT", - description: `Custom system prompt. +}; + +const getFrameworkEnvs = ( + framework?: TemplateFramework, + port?: number, +): EnvVar[] => { + if (framework !== "fastapi") { + return []; + } + return [ + { + name: "APP_HOST", + description: "The address to start the backend app.", + value: "0.0.0.0", + }, + { + name: "APP_PORT", + description: "The port to start the backend app.", + value: port?.toString() || "8000", + }, + { + name: "LLM_TEMPERATURE", + description: "Temperature for sampling from the model.", + }, + { + name: "LLM_MAX_TOKENS", + description: "Maximum number of tokens to generate.", + }, + { + name: "TOP_K", + description: + "The number of similar embeddings to return when retrieving documents.", + value: "3", + }, + { + name: "SYSTEM_PROMPT", + description: `Custom system prompt. Example: SYSTEM_PROMPT=" We have provided context information below. @@ -216,22 +211,35 @@ We have provided context information below. --------------------- Given this information, please answer the question: {query_str} "`, - }, - ], - ]; - } else { - const nextJsEnvs = [ - { - name: "NEXT_PUBLIC_MODEL", - description: "The LLM model to use (hardcode to front-end artifact).", - value: opts.model, - }, - ]; - envVars = [ - ...defaultEnvs, - ...(opts.framework === "nextjs" ? nextJsEnvs : []), - ]; - } + }, + ]; +}; + +export const createBackendEnvFile = async ( + root: string, + opts: { + llamaCloudKey?: string; + vectorDb?: TemplateVectorDB; + modelConfig: ModelConfig; + framework?: TemplateFramework; + dataSources?: TemplateDataSource[]; + port?: number; + }, +) => { + // Init env values + const envFileName = ".env"; + const envVars: EnvVar[] = [ + { + name: "LLAMA_CLOUD_API_KEY", + description: `The Llama Cloud API key.`, + value: opts.llamaCloudKey, + }, + // Add model environment variables + ...getModelEnvs(opts.modelConfig), + // Add vector database environment variables + ...getVectorDBEnvs(opts.vectorDb), + ...getFrameworkEnvs(opts.framework, opts.port), + ]; // Render and write env file const content = renderEnvVar(envVars); await fs.writeFile(path.join(root, envFileName), content); @@ -242,20 +250,9 @@ export const createFrontendEnvFile = async ( root: string, opts: { customApiPath?: string; - model?: string; }, ) => { const defaultFrontendEnvs = [ - { - name: "MODEL", - description: "The OpenAI model to use.", - value: opts.model, - }, - { - name: "NEXT_PUBLIC_MODEL", - description: "The OpenAI model to use (hardcode to front-end artifact).", - value: opts.model, - }, { name: "NEXT_PUBLIC_CHAT_API", description: "The backend API for chat endpoint.", diff --git a/helpers/index.ts b/helpers/index.ts index 8c990facb..b7b3991df 100644 --- a/helpers/index.ts +++ b/helpers/index.ts @@ -9,12 +9,14 @@ import { createBackendEnvFile, createFrontendEnvFile } from "./env-variables"; import { PackageManager } from "./get-pkg-manager"; import { installLlamapackProject } from "./llama-pack"; import { isHavingPoetryLockFile, tryPoetryRun } from "./poetry"; +import { isModelConfigured } from "./providers"; import { installPythonTemplate } from "./python"; import { downloadAndExtractRepo } from "./repo"; import { ConfigFileType, writeToolsConfig } from "./tools"; import { FileSourceConfig, InstallTemplateArgs, + ModelConfig, TemplateDataSource, TemplateFramework, TemplateVectorDB, @@ -24,8 +26,8 @@ import { installTSTemplate } from "./typescript"; // eslint-disable-next-line max-params async function generateContextData( framework: TemplateFramework, + modelConfig: ModelConfig, packageManager?: PackageManager, - openAiKey?: string, vectorDb?: TemplateVectorDB, llamaCloudKey?: string, useLlamaParse?: boolean, @@ -36,12 +38,12 @@ async function generateContextData( ? "poetry run generate" : `${packageManager} run generate`, )}`; - const openAiKeyConfigured = openAiKey || process.env["OPENAI_API_KEY"]; + const modelConfigured = isModelConfigured(modelConfig); const llamaCloudKeyConfigured = useLlamaParse ? llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"] : true; const hasVectorDb = vectorDb && vectorDb !== "none"; - if (openAiKeyConfigured && llamaCloudKeyConfigured && !hasVectorDb) { + if (modelConfigured && llamaCloudKeyConfigured && !hasVectorDb) { // If all the required environment variables are set, run the generate script if (framework === "fastapi") { if (isHavingPoetryLockFile()) { @@ -63,7 +65,7 @@ async function generateContextData( // generate the message of what to do to run the generate script manually const settings = []; - if (!openAiKeyConfigured) settings.push("your OpenAI key"); + if (!modelConfigured) settings.push("your model provider API key"); if (!llamaCloudKeyConfigured) settings.push("your Llama Cloud key"); if (hasVectorDb) settings.push("your Vector DB environment variables"); const settingsMessage = @@ -141,11 +143,9 @@ export const installTemplate = async ( // Copy the environment file to the target directory. await createBackendEnvFile(props.root, { - openAiKey: props.openAiKey, + modelConfig: props.modelConfig, llamaCloudKey: props.llamaCloudKey, vectorDb: props.vectorDb, - model: props.model, - embeddingModel: props.embeddingModel, framework: props.framework, dataSources: props.dataSources, port: props.externalPort, @@ -163,8 +163,8 @@ export const installTemplate = async ( ) { await generateContextData( props.framework, + props.modelConfig, props.packageManager, - props.openAiKey, props.vectorDb, props.llamaCloudKey, props.useLlamaParse, @@ -174,7 +174,6 @@ export const installTemplate = async ( } else { // this is a frontend for a full-stack app, create .env file with model information await createFrontendEnvFile(props.root, { - model: props.model, customApiPath: props.customApiPath, }); } diff --git a/helpers/providers/index.ts b/helpers/providers/index.ts new file mode 100644 index 000000000..bb793c090 --- /dev/null +++ b/helpers/providers/index.ts @@ -0,0 +1,66 @@ +import ciInfo from "ci-info"; +import prompts from "prompts"; +import { questionHandlers } from "../../questions"; +import { ModelConfig, ModelProvider } from "../types"; +import { askOllamaQuestions } from "./ollama"; +import { askOpenAIQuestions, isOpenAIConfigured } from "./openai"; + +const DEFAULT_MODEL_PROVIDER = "openai"; + +export type ModelConfigQuestionsParams = { + openAiKey?: string; + askModels: boolean; +}; + +export type ModelConfigParams = Omit; + +export async function askModelConfig({ + askModels, + openAiKey, +}: ModelConfigQuestionsParams): Promise { + let modelProvider: ModelProvider = DEFAULT_MODEL_PROVIDER; + if (askModels && !ciInfo.isCI) { + const { provider } = await prompts( + { + type: "select", + name: "provider", + message: "Which model provider would you like to use", + choices: [ + { + title: "OpenAI", + value: "openai", + }, + { title: "Ollama", value: "ollama" }, + ], + initial: 0, + }, + questionHandlers, + ); + modelProvider = provider; + } + + let modelConfig: ModelConfigParams; + switch (modelProvider) { + case "ollama": + modelConfig = await askOllamaQuestions({ askModels }); + break; + default: + modelConfig = await askOpenAIQuestions({ + openAiKey, + askModels, + }); + } + return { + ...modelConfig, + provider: modelProvider, + }; +} + +export function isModelConfigured(modelConfig: ModelConfig): boolean { + switch (modelConfig.provider) { + case "openai": + return isOpenAIConfigured(modelConfig); + default: + return true; + } +} diff --git a/helpers/providers/ollama.ts b/helpers/providers/ollama.ts new file mode 100644 index 000000000..a32d9d309 --- /dev/null +++ b/helpers/providers/ollama.ts @@ -0,0 +1,92 @@ +import ciInfo from "ci-info"; +import ollama, { type ModelResponse } from "ollama"; +import { red } from "picocolors"; +import prompts from "prompts"; +import { ModelConfigParams } from "."; +import { questionHandlers, toChoice } from "../../questions"; + +type ModelData = { + dimensions: number; +}; +const MODELS = ["llama3:8b", "wizardlm2:7b", "gemma:7b", "phi3"]; +const DEFAULT_MODEL = MODELS[0]; +// TODO: get embedding vector dimensions from the ollama sdk (currently not supported) +const EMBEDDING_MODELS: Record = { + "nomic-embed-text": { dimensions: 768 }, + "mxbai-embed-large": { dimensions: 1024 }, + "all-minilm": { dimensions: 384 }, +}; +const DEFAULT_EMBEDDING_MODEL: string = Object.keys(EMBEDDING_MODELS)[0]; + +type OllamaQuestionsParams = { + askModels: boolean; +}; + +export async function askOllamaQuestions({ + askModels, +}: OllamaQuestionsParams): Promise { + const config: ModelConfigParams = { + model: DEFAULT_MODEL, + embeddingModel: DEFAULT_EMBEDDING_MODEL, + dimensions: EMBEDDING_MODELS[DEFAULT_EMBEDDING_MODEL].dimensions, + }; + + // use default model values in CI or if user should not be asked + const useDefaults = ciInfo.isCI || !askModels; + if (!useDefaults) { + const { model } = await prompts( + { + type: "select", + name: "model", + message: "Which LLM model would you like to use?", + choices: MODELS.map(toChoice), + initial: 0, + }, + questionHandlers, + ); + await ensureModel(model); + config.model = model; + + const { embeddingModel } = await prompts( + { + type: "select", + name: "embeddingModel", + message: "Which embedding model would you like to use?", + choices: Object.keys(EMBEDDING_MODELS).map(toChoice), + initial: 0, + }, + questionHandlers, + ); + await ensureModel(embeddingModel); + config.embeddingModel = embeddingModel; + config.dimensions = EMBEDDING_MODELS[embeddingModel].dimensions; + } + + return config; +} + +async function ensureModel(modelName: string) { + try { + if (modelName.split(":").length === 1) { + // model doesn't have a version suffix, use latest + modelName = modelName + ":latest"; + } + const { models } = await ollama.list(); + const found = + models.find((model: ModelResponse) => model.name === modelName) !== + undefined; + if (!found) { + console.log( + red( + `Model ${modelName} was not pulled yet. Call 'ollama pull ${modelName}' and try again.`, + ), + ); + process.exit(1); + } + } catch (error) { + console.log( + red("Listing Ollama models failed. Is 'ollama' running? " + error), + ); + process.exit(1); + } +} diff --git a/helpers/providers/openai.ts b/helpers/providers/openai.ts new file mode 100644 index 000000000..2e13d99f0 --- /dev/null +++ b/helpers/providers/openai.ts @@ -0,0 +1,146 @@ +import ciInfo from "ci-info"; +import got from "got"; +import ora from "ora"; +import { red } from "picocolors"; +import prompts from "prompts"; +import { ModelConfigParams, ModelConfigQuestionsParams } from "."; +import { questionHandlers } from "../../questions"; + +const OPENAI_API_URL = "https://api.openai.com/v1"; + +const DEFAULT_MODEL = "gpt-4-turbo"; +const DEFAULT_EMBEDDING_MODEL = "text-embedding-3-large"; + +export async function askOpenAIQuestions({ + openAiKey, + askModels, +}: ModelConfigQuestionsParams): Promise { + const config: ModelConfigParams = { + apiKey: openAiKey, + model: DEFAULT_MODEL, + embeddingModel: DEFAULT_EMBEDDING_MODEL, + dimensions: getDimensions(DEFAULT_EMBEDDING_MODEL), + }; + + if (!config.apiKey) { + const { key } = await prompts( + { + type: "text", + name: "key", + message: askModels + ? "Please provide your OpenAI API key (or leave blank to use OPENAI_API_KEY env variable):" + : "Please provide your OpenAI API key (leave blank to skip):", + validate: (value: string) => { + console.log(value); + if (askModels && !value) { + if (process.env.OPENAI_API_KEY) { + return true; + } + return "OPENAI_API_KEY env variable is not set - key is required"; + } + return true; + }, + }, + questionHandlers, + ); + config.apiKey = key || process.env.OPENAI_API_KEY; + } + + // use default model values in CI or if user should not be asked + const useDefaults = ciInfo.isCI || !askModels; + if (!useDefaults) { + const { model } = await prompts( + { + type: "select", + name: "model", + message: "Which LLM model would you like to use?", + choices: await getAvailableModelChoices(false, config.apiKey), + initial: 0, + }, + questionHandlers, + ); + config.model = model; + + const { embeddingModel } = await prompts( + { + type: "select", + name: "embeddingModel", + message: "Which embedding model would you like to use?", + choices: await getAvailableModelChoices(true, config.apiKey), + initial: 0, + }, + questionHandlers, + ); + config.embeddingModel = embeddingModel; + config.dimensions = getDimensions(embeddingModel); + } + + return config; +} + +export function isOpenAIConfigured(params: ModelConfigParams): boolean { + if (params.apiKey) { + return true; + } + if (process.env["OPENAI_API_KEY"]) { + return true; + } + return false; +} + +async function getAvailableModelChoices( + selectEmbedding: boolean, + apiKey?: string, +) { + if (!apiKey) { + throw new Error("need OpenAI key to retrieve model choices"); + } + const isLLMModel = (modelId: string) => { + return modelId.startsWith("gpt"); + }; + + const isEmbeddingModel = (modelId: string) => { + return modelId.includes("embedding"); + }; + + const spinner = ora("Fetching available models").start(); + try { + const response = await got(`${OPENAI_API_URL}/models`, { + headers: { + Authorization: "Bearer " + apiKey, + }, + timeout: 5000, + responseType: "json", + }); + const data: any = await response.body; + spinner.stop(); + return data.data + .filter((model: any) => + selectEmbedding ? isEmbeddingModel(model.id) : isLLMModel(model.id), + ) + .map((el: any) => { + return { + title: el.id, + value: el.id, + }; + }); + } catch (error) { + spinner.stop(); + if ((error as any).response?.statusCode === 401) { + console.log( + red( + "Invalid OpenAI API key provided! Please provide a valid key and try again!", + ), + ); + } else { + console.log(red("Request failed: " + error)); + } + process.exit(1); + } +} + +function getDimensions(modelName: string) { + // at 2024-04-24 all OpenAI embedding models support 1536 dimensions except + // "text-embedding-3-large", see https://openai.com/blog/new-embedding-models-and-api-updates + return modelName === "text-embedding-3-large" ? 1024 : 1536; +} diff --git a/helpers/python.ts b/helpers/python.ts index af83dd2c8..316fe52d5 100644 --- a/helpers/python.ts +++ b/helpers/python.ts @@ -10,6 +10,7 @@ import { isPoetryAvailable, tryPoetryInstall } from "./poetry"; import { Tool } from "./tools"; import { InstallTemplateArgs, + ModelConfig, TemplateDataSource, TemplateVectorDB, } from "./types"; @@ -21,6 +22,7 @@ interface Dependency { } const getAdditionalDependencies = ( + modelConfig: ModelConfig, vectorDb?: TemplateVectorDB, dataSource?: TemplateDataSource, tools?: Tool[], @@ -108,6 +110,25 @@ const getAdditionalDependencies = ( }); }); + switch (modelConfig.provider) { + case "ollama": + dependencies.push({ + name: "llama-index-llms-ollama", + version: "0.1.2", + }); + dependencies.push({ + name: "llama-index-embeddings-ollama", + version: "0.1.2", + }); + break; + case "openai": + dependencies.push({ + name: "llama-index-agent-openai", + version: "0.2.2", + }); + break; + } + return dependencies; }; @@ -205,8 +226,8 @@ export const installPythonTemplate = async ({ dataSources, tools, postInstallAction, - useLlamaParse, observability, + modelConfig, }: Pick< InstallTemplateArgs, | "root" @@ -215,9 +236,9 @@ export const installPythonTemplate = async ({ | "vectorDb" | "dataSources" | "tools" - | "useLlamaParse" | "postInstallAction" | "observability" + | "modelConfig" >) => { console.log("\nInitializing Python project with template:", template, "\n"); const templatePath = path.join(templatesDir, "types", template, framework); @@ -258,7 +279,7 @@ export const installPythonTemplate = async ({ }); const addOnDependencies = dataSources - .map((ds) => getAdditionalDependencies(vectorDb, ds, tools)) + .map((ds) => getAdditionalDependencies(modelConfig, vectorDb, ds, tools)) .flat(); if (observability === "opentelemetry") { diff --git a/helpers/types.ts b/helpers/types.ts index 0db557ed1..42b571e7f 100644 --- a/helpers/types.ts +++ b/helpers/types.ts @@ -1,6 +1,14 @@ import { PackageManager } from "../helpers/get-pkg-manager"; import { Tool } from "./tools"; +export type ModelProvider = "openai" | "ollama"; +export type ModelConfig = { + provider: ModelProvider; + apiKey?: string; + model: string; + embeddingModel: string; + dimensions: number; +}; export type TemplateType = "streaming" | "community" | "llamapack"; export type TemplateFramework = "nextjs" | "express" | "fastapi"; export type TemplateUI = "html" | "shadcn"; @@ -59,11 +67,9 @@ export interface InstallTemplateArgs { ui: TemplateUI; dataSources: TemplateDataSource[]; customApiPath?: string; - openAiKey?: string; + modelConfig: ModelConfig; llamaCloudKey?: string; useLlamaParse?: boolean; - model: string; - embeddingModel: string; communityProjectConfig?: CommunityProjectConfig; llamapack?: string; vectorDb?: TemplateVectorDB; diff --git a/index.ts b/index.ts index 45b79b38d..11f3d83dd 100644 --- a/index.ts +++ b/index.ts @@ -107,20 +107,6 @@ const program = new Commander.Command(packageJson.name) ` Whether to generate a frontend for your backend. -`, - ) - .option( - "--model ", - ` - - Select OpenAI model to use. E.g. gpt-3.5-turbo. -`, - ) - .option( - "--embedding-model ", - ` - - Select OpenAI embedding model to use. E.g. text-embedding-ada-002. `, ) .option( @@ -201,9 +187,7 @@ if (process.argv.includes("--tools")) { if (process.argv.includes("--no-llama-parse")) { program.useLlamaParse = false; } -if (process.argv.includes("--ask-models")) { - program.askModels = true; -} +program.askModels = process.argv.includes("--ask-models"); if (process.argv.includes("--no-files")) { program.dataSources = []; } else { @@ -290,7 +274,11 @@ async function run(): Promise { } const preferences = (conf.get("preferences") || {}) as QuestionArgs; - await askQuestions(program as unknown as QuestionArgs, preferences); + await askQuestions( + program as unknown as QuestionArgs, + preferences, + program.openAiKey, + ); await createApp({ template: program.template, @@ -299,10 +287,8 @@ async function run(): Promise { appPath: resolvedProjectPath, packageManager, frontend: program.frontend, - openAiKey: program.openAiKey, + modelConfig: program.modelConfig, llamaCloudKey: program.llamaCloudKey, - model: program.model, - embeddingModel: program.embeddingModel, communityProjectConfig: program.communityProjectConfig, llamapack: program.llamapack, vectorDb: program.vectorDb, diff --git a/package.json b/package.json index 6cb3bd72d..867f4f4b7 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { "name": "create-llama", "version": "0.0.32", + "description": "Create LlamaIndex-powered apps with one command", "keywords": [ "rag", "llamaindex", "next.js" ], - "description": "Create LlamaIndex-powered apps with one command", "repository": { "type": "git", "url": "https://github.com/run-llama/LlamaIndexTS", @@ -20,32 +20,30 @@ "dist" ], "scripts": { + "build": "bash ./scripts/build.sh", + "build:ncc": "pnpm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register", "clean": "rimraf --glob ./dist ./templates/**/__pycache__ ./templates/**/node_modules ./templates/**/poetry.lock", + "dev": "ncc build ./index.ts -w -o dist/", + "e2e": "playwright test", "format": "prettier --ignore-unknown --cache --check .", "format:write": "prettier --ignore-unknown --write .", - "dev": "ncc build ./index.ts -w -o dist/", - "build": "bash ./scripts/build.sh", - "build:ncc": "pnpm run clean && ncc build ./index.ts -o ./dist/ --minify --no-cache --no-source-map-register", "lint": "eslint . --ignore-pattern dist --ignore-pattern e2e/cache", - "e2e": "playwright test", + "new-snapshot": "pnpm run build && changeset version --snapshot", + "new-version": "pnpm run build && changeset version", + "pack-install": "bash ./scripts/pack.sh", "prepare": "husky", "release": "pnpm run build && changeset publish", - "new-version": "pnpm run build && changeset version", - "release-snapshot": "pnpm run build && changeset publish --tag snapshot", - "new-snapshot": "pnpm run build && changeset version --snapshot", - "pack-install": "bash ./scripts/pack.sh" + "release-snapshot": "pnpm run build && changeset publish --tag snapshot" }, - "devDependencies": { - "@playwright/test": "^1.41.1", + "dependencies": { "@types/async-retry": "1.4.2", "@types/ci-info": "2.0.0", "@types/cross-spawn": "6.0.0", + "@types/fs-extra": "11.0.4", "@types/node": "^20.11.7", "@types/prompts": "2.0.1", "@types/tar": "6.1.5", "@types/validate-npm-package-name": "3.0.0", - "@types/fs-extra": "11.0.4", - "@vercel/ncc": "0.38.1", "async-retry": "1.3.1", "async-sema": "3.0.1", "ci-info": "github:watson/ci-info#f43f6a1cefff47fb361c88cf4b943fdbcaafe540", @@ -53,29 +51,34 @@ "conf": "10.2.0", "cross-spawn": "7.0.3", "fast-glob": "3.3.1", + "fs-extra": "11.2.0", "got": "10.7.0", + "ollama": "^0.5.0", + "ora": "^8.0.1", "picocolors": "1.0.0", "prompts": "2.1.0", - "rimraf": "^5.0.5", "smol-toml": "^1.1.4", "tar": "6.1.15", "terminal-link": "^3.0.0", "update-check": "1.5.4", "validate-npm-package-name": "3.0.0", - "wait-port": "^1.1.0", + "yaml": "2.4.1" + }, + "devDependencies": { "@changesets/cli": "^2.27.1", + "@playwright/test": "^1.41.1", + "@vercel/ncc": "0.38.1", "eslint": "^8.56.0", + "eslint-config-prettier": "^8.10.0", "husky": "^9.0.10", "prettier": "^3.2.5", "prettier-plugin-organize-imports": "^3.2.4", + "rimraf": "^5.0.5", "typescript": "^5.3.3", - "eslint-config-prettier": "^8.10.0", - "ora": "^8.0.1", - "fs-extra": "11.2.0", - "yaml": "2.4.1" + "wait-port": "^1.1.0" }, + "packageManager": "pnpm@9.0.5", "engines": { "node": ">=16.14.0" - }, - "packageManager": "pnpm@9.0.5" + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06852a768..3d3e9b2d3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,13 +7,7 @@ settings: importers: .: - devDependencies: - '@changesets/cli': - specifier: ^2.27.1 - version: 2.27.1 - '@playwright/test': - specifier: ^1.41.1 - version: 1.42.1 + dependencies: '@types/async-retry': specifier: 1.4.2 version: 1.4.2 @@ -38,9 +32,6 @@ importers: '@types/validate-npm-package-name': specifier: 3.0.0 version: 3.0.0 - '@vercel/ncc': - specifier: 0.38.1 - version: 0.38.1 async-retry: specifier: 1.3.1 version: 1.3.1 @@ -59,12 +50,6 @@ importers: cross-spawn: specifier: 7.0.3 version: 7.0.3 - eslint: - specifier: ^8.56.0 - version: 8.57.0 - eslint-config-prettier: - specifier: ^8.10.0 - version: 8.10.0(eslint@8.57.0) fast-glob: specifier: 3.3.1 version: 3.3.1 @@ -74,27 +59,18 @@ importers: got: specifier: 10.7.0 version: 10.7.0 - husky: - specifier: ^9.0.10 - version: 9.0.11 + ollama: + specifier: ^0.5.0 + version: 0.5.0 ora: specifier: ^8.0.1 version: 8.0.1 picocolors: specifier: 1.0.0 version: 1.0.0 - prettier: - specifier: ^3.2.5 - version: 3.2.5 - prettier-plugin-organize-imports: - specifier: ^3.2.4 - version: 3.2.4(prettier@3.2.5)(typescript@5.4.2) prompts: specifier: 2.1.0 version: 2.1.0 - rimraf: - specifier: ^5.0.5 - version: 5.0.5 smol-toml: specifier: ^1.1.4 version: 1.1.4 @@ -104,21 +80,49 @@ importers: terminal-link: specifier: ^3.0.0 version: 3.0.0 - typescript: - specifier: ^5.3.3 - version: 5.4.2 update-check: specifier: 1.5.4 version: 1.5.4 validate-npm-package-name: specifier: 3.0.0 version: 3.0.0 - wait-port: - specifier: ^1.1.0 - version: 1.1.0 yaml: specifier: 2.4.1 version: 2.4.1 + devDependencies: + '@changesets/cli': + specifier: ^2.27.1 + version: 2.27.1 + '@playwright/test': + specifier: ^1.41.1 + version: 1.42.1 + '@vercel/ncc': + specifier: 0.38.1 + version: 0.38.1 + eslint: + specifier: ^8.56.0 + version: 8.57.0 + eslint-config-prettier: + specifier: ^8.10.0 + version: 8.10.0(eslint@8.57.0) + husky: + specifier: ^9.0.10 + version: 9.0.11 + prettier: + specifier: ^3.2.5 + version: 3.2.5 + prettier-plugin-organize-imports: + specifier: ^3.2.4 + version: 3.2.4(prettier@3.2.5)(typescript@5.4.2) + rimraf: + specifier: ^5.0.5 + version: 5.0.5 + typescript: + specifier: ^5.3.3 + version: 5.4.2 + wait-port: + specifier: ^1.1.0 + version: 1.1.0 packages: @@ -1287,6 +1291,9 @@ packages: resolution: {integrity: sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==} engines: {node: '>= 0.4'} + ollama@0.5.0: + resolution: {integrity: sha512-CRtRzsho210EGdK52GrUMohA2pU+7NbgEaBG3DcYeRmvQthDO7E2LHOkLlUUeaYUlNmEd8icbjC02ug9meSYnw==} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -1844,6 +1851,9 @@ packages: wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + which-boxed-primitive@1.0.2: resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==} @@ -3256,6 +3266,10 @@ snapshots: has-symbols: 1.0.3 object-keys: 1.1.1 + ollama@0.5.0: + dependencies: + whatwg-fetch: 3.6.20 + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -3816,6 +3830,8 @@ snapshots: dependencies: defaults: 1.0.4 + whatwg-fetch@3.6.20: {} + which-boxed-primitive@1.0.2: dependencies: is-bigint: 1.0.4 diff --git a/questions.ts b/questions.ts index 375ad8078..31eed2247 100644 --- a/questions.ts +++ b/questions.ts @@ -1,8 +1,6 @@ import { execSync } from "child_process"; import ciInfo from "ci-info"; import fs from "fs"; -import got from "got"; -import ora from "ora"; import path from "path"; import { blue, green, red } from "picocolors"; import prompts from "prompts"; @@ -16,11 +14,10 @@ import { COMMUNITY_OWNER, COMMUNITY_REPO } from "./helpers/constant"; import { EXAMPLE_FILE } from "./helpers/datasources"; import { templatesDir } from "./helpers/dir"; import { getAvailableLlamapackOptions } from "./helpers/llama-pack"; +import { askModelConfig, isModelConfigured } from "./helpers/providers"; import { getProjectOptions } from "./helpers/repo"; import { supportedTools, toolsRequireConfig } from "./helpers/tools"; -const OPENAI_API_URL = "https://api.openai.com/v1"; - export type QuestionArgs = Omit< InstallAppArgs, "appPath" | "packageManager" @@ -67,16 +64,13 @@ if ($dialogResult -eq [System.Windows.Forms.DialogResult]::OK) } `; -const defaults: QuestionArgs = { +const defaults: Omit = { template: "streaming", framework: "nextjs", ui: "shadcn", frontend: false, - openAiKey: "", llamaCloudKey: "", useLlamaParse: false, - model: "gpt-4-turbo", - embeddingModel: "text-embedding-3-large", communityProjectConfig: undefined, llamapack: "", postInstallAction: "dependencies", @@ -84,7 +78,7 @@ const defaults: QuestionArgs = { tools: [], }; -const handlers = { +export const questionHandlers = { onCancel: () => { console.error("Exiting."); process.exit(1); @@ -232,63 +226,15 @@ export const onPromptState = (state: any) => { } }; -const getAvailableModelChoices = async ( - selectEmbedding: boolean, - apiKey?: string, -) => { - const isLLMModel = (modelId: string) => { - return modelId.startsWith("gpt"); - }; - - const isEmbeddingModel = (modelId: string) => { - return modelId.includes("embedding"); - }; - - if (apiKey) { - const spinner = ora("Fetching available models").start(); - try { - const response = await got(`${OPENAI_API_URL}/models`, { - headers: { - Authorization: "Bearer " + apiKey, - }, - timeout: 5000, - responseType: "json", - }); - const data: any = await response.body; - spinner.stop(); - return data.data - .filter((model: any) => - selectEmbedding ? isEmbeddingModel(model.id) : isLLMModel(model.id), - ) - .map((el: any) => { - return { - title: el.id, - value: el.id, - }; - }); - } catch (error) { - spinner.stop(); - if ((error as any).response?.statusCode === 401) { - console.log( - red( - "Invalid OpenAI API key provided! Please provide a valid key and try again!", - ), - ); - } else { - console.log(red("Request failed: " + error)); - } - process.exit(1); - } - } -}; - export const askQuestions = async ( program: QuestionArgs, preferences: QuestionArgs, + openAiKey?: string, ) => { - const getPrefOrDefault = ( + const getPrefOrDefault = >( field: K, - ): QuestionArgs[K] => preferences[field] ?? defaults[field]; + ): Omit[K] => + preferences[field] ?? defaults[field]; // Ask for next action after installation async function askPostInstallAction() { @@ -311,8 +257,7 @@ export const askQuestions = async ( }, ]; - const openAiKeyConfigured = - program.openAiKey || process.env["OPENAI_API_KEY"]; + const modelConfigured = isModelConfigured(program.modelConfig); // If using LlamaParse, require LlamaCloud API key const llamaCloudKeyConfigured = program.useLlamaParse ? program.llamaCloudKey || process.env["LLAMA_CLOUD_API_KEY"] @@ -321,7 +266,7 @@ export const askQuestions = async ( // Can run the app if all tools do not require configuration if ( !hasVectorDb && - openAiKeyConfigured && + modelConfigured && llamaCloudKeyConfigured && !toolsRequireConfig(program.tools) && !program.llamapack @@ -341,7 +286,7 @@ export const askQuestions = async ( choices: actionChoices, initial: 1, }, - handlers, + questionHandlers, ); program.postInstallAction = action; @@ -374,7 +319,7 @@ export const askQuestions = async ( ], initial: 0, }, - handlers, + questionHandlers, ); program.template = template; preferences.template = template; @@ -397,7 +342,7 @@ export const askQuestions = async ( })), initial: 0, }, - handlers, + questionHandlers, ); const projectConfig = JSON.parse(communityProjectConfig); program.communityProjectConfig = projectConfig; @@ -418,7 +363,7 @@ export const askQuestions = async ( })), initial: 0, }, - handlers, + questionHandlers, ); program.llamapack = llamapack; preferences.llamapack = llamapack; @@ -444,7 +389,7 @@ export const askQuestions = async ( choices, initial: 0, }, - handlers, + questionHandlers, ); program.framework = framework; preferences.framework = framework; @@ -504,7 +449,7 @@ export const askQuestions = async ( ], initial: 0, }, - handlers, + questionHandlers, ); program.observability = observability; @@ -512,67 +457,13 @@ export const askQuestions = async ( } } - if (!program.openAiKey) { - const { key } = await prompts( - { - type: "text", - name: "key", - message: program.askModels - ? "Please provide your OpenAI API key (or leave blank to reuse OPENAI_API_KEY env variable):" - : "Please provide your OpenAI API key (leave blank to skip):", - validate: (value: string) => { - if (program.askModels && !value) { - if (process.env.OPENAI_API_KEY) { - return true; - } - return "OpenAI API key is required"; - } - return true; - }, - }, - handlers, - ); - - program.openAiKey = key || process.env.OPENAI_API_KEY; - preferences.openAiKey = key || process.env.OPENAI_API_KEY; - } - - if (!program.model) { - if (ciInfo.isCI || !program.askModels) { - program.model = defaults.model; - } else { - const { model } = await prompts( - { - type: "select", - name: "model", - message: "Which LLM model would you like to use?", - choices: await getAvailableModelChoices(false, program.openAiKey), - initial: 0, - }, - handlers, - ); - program.model = model; - preferences.model = model; - } - } - - if (!program.embeddingModel) { - if (ciInfo.isCI || !program.askModels) { - program.embeddingModel = defaults.embeddingModel; - } else { - const { embeddingModel } = await prompts( - { - type: "select", - name: "embeddingModel", - message: "Which embedding model would you like to use?", - choices: await getAvailableModelChoices(true, program.openAiKey), - initial: 0, - }, - handlers, - ); - program.embeddingModel = embeddingModel; - preferences.embeddingModel = embeddingModel; - } + if (!program.modelConfig) { + const modelConfig = await askModelConfig({ + openAiKey, + askModels: program.askModels ?? false, + }); + program.modelConfig = modelConfig; + preferences.modelConfig = modelConfig; } if (!program.dataSources) { @@ -596,7 +487,7 @@ export const askQuestions = async ( ), initial: firstQuestion ? 1 : 0, }, - handlers, + questionHandlers, ); if (selectedSource === "no" || selectedSource === "none") { @@ -642,7 +533,7 @@ export const askQuestions = async ( return true; }, }, - handlers, + questionHandlers, ); program.dataSources.push({ @@ -687,7 +578,7 @@ export const askQuestions = async ( ]; program.dataSources.push({ type: "db", - config: await prompts(dbPrompts, handlers), + config: await prompts(dbPrompts, questionHandlers), }); } } @@ -714,7 +605,7 @@ export const askQuestions = async ( active: "yes", inactive: "no", }, - handlers, + questionHandlers, ); program.useLlamaParse = useLlamaParse; @@ -727,7 +618,7 @@ export const askQuestions = async ( message: "Please provide your LlamaIndex Cloud API key (leave blank to skip):", }, - handlers, + questionHandlers, ); program.llamaCloudKey = llamaCloudKey; } @@ -746,7 +637,7 @@ export const askQuestions = async ( choices: getVectorDbChoices(program.framework), initial: 0, }, - handlers, + questionHandlers, ); program.vectorDb = vectorDb; preferences.vectorDb = vectorDb; @@ -781,3 +672,7 @@ export const askQuestions = async ( await askPostInstallAction(); }; + +export const toChoice = (value: string) => { + return { title: value, value }; +}; diff --git a/templates/components/vectordbs/typescript/pg/shared.ts b/templates/components/vectordbs/typescript/pg/shared.ts index 88774df12..e241f9070 100644 --- a/templates/components/vectordbs/typescript/pg/shared.ts +++ b/templates/components/vectordbs/typescript/pg/shared.ts @@ -2,7 +2,7 @@ export const PGVECTOR_COLLECTION = "data"; export const PGVECTOR_SCHEMA = "public"; export const PGVECTOR_TABLE = "llamaindex_embedding"; -const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING", "OPENAI_API_KEY"]; +const REQUIRED_ENV_VARS = ["PG_CONNECTION_STRING"]; export function checkRequiredEnvVars() { const missingEnvVars = REQUIRED_ENV_VARS.filter((envVar) => { diff --git a/templates/types/streaming/express/package.json b/templates/types/streaming/express/package.json index 834841f53..612076e54 100644 --- a/templates/types/streaming/express/package.json +++ b/templates/types/streaming/express/package.json @@ -14,7 +14,7 @@ "cors": "^2.8.5", "dotenv": "^16.3.1", "express": "^4.18.2", - "llamaindex": "0.2.9" + "llamaindex": "0.2.10" }, "devDependencies": { "@types/cors": "^2.8.16", diff --git a/templates/types/streaming/express/src/controllers/engine/settings.ts b/templates/types/streaming/express/src/controllers/engine/settings.ts index 1f5f397a4..258043993 100644 --- a/templates/types/streaming/express/src/controllers/engine/settings.ts +++ b/templates/types/streaming/express/src/controllers/engine/settings.ts @@ -1,19 +1,52 @@ -import { OpenAI, OpenAIEmbedding, Settings } from "llamaindex"; +import { + Ollama, + OllamaEmbedding, + OpenAI, + OpenAIEmbedding, + Settings, +} from "llamaindex"; const CHUNK_SIZE = 512; const CHUNK_OVERLAP = 20; export const initSettings = async () => { + // HINT: you can delete the initialization code for unused model providers + console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`); + switch (process.env.MODEL_PROVIDER) { + case "ollama": + initOllama(); + break; + default: + initOpenAI(); + break; + } + Settings.chunkSize = CHUNK_SIZE; + Settings.chunkOverlap = CHUNK_OVERLAP; +}; + +function initOpenAI() { Settings.llm = new OpenAI({ model: process.env.MODEL ?? "gpt-3.5-turbo", maxTokens: 512, }); - Settings.chunkSize = CHUNK_SIZE; - Settings.chunkOverlap = CHUNK_OVERLAP; Settings.embedModel = new OpenAIEmbedding({ model: process.env.EMBEDDING_MODEL, dimensions: process.env.EMBEDDING_DIM ? parseInt(process.env.EMBEDDING_DIM) : undefined, }); -}; +} + +function initOllama() { + if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) { + throw new Error( + "Using Ollama as model provider, 'MODEL' and 'EMBEDDING_MODEL' env variables must be set.", + ); + } + Settings.llm = new Ollama({ + model: process.env.MODEL ?? "", + }); + Settings.embedModel = new OllamaEmbedding({ + model: process.env.EMBEDDING_MODEL ?? "", + }); +} diff --git a/templates/types/streaming/fastapi/README-template.md b/templates/types/streaming/fastapi/README-template.md index 6af9712b4..7969ff0ea 100644 --- a/templates/types/streaming/fastapi/README-template.md +++ b/templates/types/streaming/fastapi/README-template.md @@ -11,13 +11,7 @@ poetry install poetry shell ``` -By default, we use the OpenAI LLM (though you can customize, see `app/settings.py`). As a result, you need to specify an `OPENAI_API_KEY` in an .env file in this directory. - -Example `.env` file: - -``` -OPENAI_API_KEY= -``` +Then check the parameters that have been pre-configured in the `.env` file in this directory. (E.g. you might need to configure an `OPENAI_API_KEY` if you're using OpenAI as model provider). If you are using any tools or data sources, you can update their config files in the `config` folder. diff --git a/templates/types/streaming/fastapi/app/settings.py b/templates/types/streaming/fastapi/app/settings.py index 3f2c5e078..be272d548 100644 --- a/templates/types/streaming/fastapi/app/settings.py +++ b/templates/types/streaming/fastapi/app/settings.py @@ -1,41 +1,44 @@ import os from typing import Dict from llama_index.core.settings import Settings -from llama_index.llms.openai import OpenAI -from llama_index.embeddings.openai import OpenAIEmbedding -def llm_config_from_env() -> Dict: +def init_settings(): + model_provider = os.getenv("MODEL_PROVIDER") + if model_provider == "openai": + init_openai() + elif model_provider == "ollama": + init_ollama() + else: + raise ValueError(f"Invalid model provider: {model_provider}") + Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024")) + Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20")) + + +def init_ollama(): + from llama_index.llms.ollama import Ollama + from llama_index.embeddings.ollama import OllamaEmbedding + + Settings.embed_model = OllamaEmbedding(model_name=os.getenv("EMBEDDING_MODEL")) + Settings.llm = Ollama(model=os.getenv("MODEL")) + + +def init_openai(): + from llama_index.llms.openai import OpenAI + from llama_index.embeddings.openai import OpenAIEmbedding from llama_index.core.constants import DEFAULT_TEMPERATURE - model = os.getenv("MODEL") - temperature = os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE) max_tokens = os.getenv("LLM_MAX_TOKENS") - config = { - "model": model, - "temperature": float(temperature), + "model": os.getenv("MODEL"), + "temperature": float(os.getenv("LLM_TEMPERATURE", DEFAULT_TEMPERATURE)), "max_tokens": int(max_tokens) if max_tokens is not None else None, } - return config + Settings.llm = OpenAI(**config) - -def embedding_config_from_env() -> Dict: - model = os.getenv("EMBEDDING_MODEL") dimension = os.getenv("EMBEDDING_DIM") - config = { - "model": model, + "model": os.getenv("EMBEDDING_MODEL"), "dimension": int(dimension) if dimension is not None else None, } - return config - - -def init_settings(): - llm_configs = llm_config_from_env() - embedding_configs = embedding_config_from_env() - - Settings.llm = OpenAI(**llm_configs) - Settings.embed_model = OpenAIEmbedding(**embedding_configs) - Settings.chunk_size = int(os.getenv("CHUNK_SIZE", "1024")) - Settings.chunk_overlap = int(os.getenv("CHUNK_OVERLAP", "20")) + Settings.embed_model = OpenAIEmbedding(**config) diff --git a/templates/types/streaming/fastapi/pyproject.toml b/templates/types/streaming/fastapi/pyproject.toml index 30cf267f1..93176212b 100644 --- a/templates/types/streaming/fastapi/pyproject.toml +++ b/templates/types/streaming/fastapi/pyproject.toml @@ -15,7 +15,6 @@ uvicorn = { extras = ["standard"], version = "^0.23.2" } python-dotenv = "^1.0.0" llama-index = "0.10.28" llama-index-core = "0.10.28" -llama-index-agent-openai = "0.2.2" [build-system] requires = ["poetry-core"] diff --git a/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts index 1f5f397a4..258043993 100644 --- a/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts +++ b/templates/types/streaming/nextjs/app/api/chat/engine/settings.ts @@ -1,19 +1,52 @@ -import { OpenAI, OpenAIEmbedding, Settings } from "llamaindex"; +import { + Ollama, + OllamaEmbedding, + OpenAI, + OpenAIEmbedding, + Settings, +} from "llamaindex"; const CHUNK_SIZE = 512; const CHUNK_OVERLAP = 20; export const initSettings = async () => { + // HINT: you can delete the initialization code for unused model providers + console.log(`Using '${process.env.MODEL_PROVIDER}' model provider`); + switch (process.env.MODEL_PROVIDER) { + case "ollama": + initOllama(); + break; + default: + initOpenAI(); + break; + } + Settings.chunkSize = CHUNK_SIZE; + Settings.chunkOverlap = CHUNK_OVERLAP; +}; + +function initOpenAI() { Settings.llm = new OpenAI({ model: process.env.MODEL ?? "gpt-3.5-turbo", maxTokens: 512, }); - Settings.chunkSize = CHUNK_SIZE; - Settings.chunkOverlap = CHUNK_OVERLAP; Settings.embedModel = new OpenAIEmbedding({ model: process.env.EMBEDDING_MODEL, dimensions: process.env.EMBEDDING_DIM ? parseInt(process.env.EMBEDDING_DIM) : undefined, }); -}; +} + +function initOllama() { + if (!process.env.MODEL || !process.env.EMBEDDING_MODEL) { + throw new Error( + "Using Ollama as model provider, 'MODEL' and 'EMBEDDING_MODEL' env variables must be set.", + ); + } + Settings.llm = new Ollama({ + model: process.env.MODEL ?? "", + }); + Settings.embedModel = new OllamaEmbedding({ + model: process.env.EMBEDDING_MODEL ?? "", + }); +} diff --git a/templates/types/streaming/nextjs/app/components/chat-section.tsx b/templates/types/streaming/nextjs/app/components/chat-section.tsx index e98607462..afb59960f 100644 --- a/templates/types/streaming/nextjs/app/components/chat-section.tsx +++ b/templates/types/streaming/nextjs/app/components/chat-section.tsx @@ -36,7 +36,7 @@ export default function ChatSection() { handleSubmit={handleSubmit} handleInputChange={handleInputChange} isLoading={isLoading} - multiModal={process.env.NEXT_PUBLIC_MODEL === "gpt-4-turbo"} + multiModal={true} /> ); diff --git a/templates/types/streaming/nextjs/package.json b/templates/types/streaming/nextjs/package.json index baddb358c..221c6998f 100644 --- a/templates/types/streaming/nextjs/package.json +++ b/templates/types/streaming/nextjs/package.json @@ -16,7 +16,7 @@ "class-variance-authority": "^0.7.0", "clsx": "^1.2.1", "dotenv": "^16.3.1", - "llamaindex": "0.2.9", + "llamaindex": "0.2.10", "lucide-react": "^0.294.0", "next": "^14.0.3", "react": "^18.2.0",