init project

.devcontainer/devcontainer.json

@@ -0,0 +1,41 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+	"features": {
+		"ghcr.io/devcontainers-contrib/features/poetry:2": {},
+		"ghcr.io/devcontainers/features/docker-in-docker": {}
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-toolsai.jupyter",
+				"GitHub.copilot-chat",
+				"GitHub.copilot",
+				"GitHub.vscode-pull-request-github",
+				"eamodio.gitlens"
+			]
+		}
+	},
+	"containerEnv": {
+		"POETRY_VIRTUALENVS_IN_PROJECT": "true"
+	}
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/st-weaviate-connection.code-workspace

@@ -0,0 +1,13 @@
+{
+    "folders": [
+		{
+			"path": ".."
+		},
+		{
+			"path": "../.venv/lib/python3.11/site-packages/weaviate"
+		},
+		{
+			"path": "../.venv/lib/python3.11/site-packages/streamlit"
+		}
+	]
+}

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+.DS_Store
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Weaviate
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -1 +1,85 @@
-# weaviate-magic-chat-demo
+# Streamlit-Weaviate Connection
+
+[![Weaviate](https://img.shields.io/static/v1?label=Built%20with&message=Weaviate&color=green&style=flat-square)](https://weaviate.io/) [![Weaviate](https://img.shields.io/static/v1?label=%20made%20with%20%E2%9D%A4%20for&message=Streamlit&color=red&style=flat-square)](https://weaviate.io/)
+
+This project is a submission for the [Streamlit Connections Hackathon 2023](https://discuss.streamlit.io/t/connections-hackathon/47574).
+It delivers a Streamlit connector for the open-source vector database, [Weaviate](https://weaviate.io/).
+
+## Overview
+
+The Streamlit-Weaviate Connector enables developers to connect to a Weaviate database with the following Python code:
+
+ ```python 
+    conn = st.experimental_connection(
+        "weaviate",
+        type=WeaviateConnection,
+        url=os.getenv("WEAVIATE_URL"),
+        api_key=os.getenv("WEAVIATE_API_KEY"),
+    )
+ ```
+
+This project also includes a Streamlit demo, "Magic Chat", designed to search through [Magic The Gathering](https://magic.wizards.com/en) cards with various search options, such as BM25, Semantic Search, Hybrid Search and Generative Search. The live demo is accessible through [Streamlit Community Cloud](https://streamlit.io/cloud)
+
+![Screenshot of the demo](https://github.com/weaviate/st-weaviate-connection/blob/main/img/screenshot.jpeg)
+
+# 📚 Quickstart Guide
+## 🔧 Installation
+
+This project uses `poetry` for dependency management. You can find more details more `poetry` in [its documentation](https://python-poetry.org/docs/dependency-specification/).
+
+1. **Create a new Python virtual environment:**
+- Ensure you have python `>=3.10.0` installed
+- ```python3 -m venv env```
+- ```source env/bin/activate```
+- ```pip install poetry```
+
+2. **Install the project:**
+- Install the project using poetry
+-  ```bash 
+        poetry add git+https://github.com/weaviate/st-weaviate-connection.git
+    ```
+
+## 🔗 Basic Usage
+
+The project includes a demonstration notebook to showcase basic functionalities of the connector (see here [demo notebook](./notebooks/01_demo.ipynb)) and a [streamlit app](./streamlit_app.py) illustrating the implementation and usage of the connector.
+
+Before you run the Jupyter notebook or the Streamlit app, create a `.env` file in the root directory of the project and add your Weaviate cluster and OpenAI API credentials:
+
+**Set environment variables:**
+```
+WEAVIATE_URL= YOUR WEAVIATE_CLUSTER_URL
+WEAVIATE_API_KEY= YOUR WEAVIATE_API_KEY
+OPENAI_API_KEY= (ONLY NEEDED FOR STREAMLIT APP)
+```
+
+To set up your Weaviate cluster, follow either of these methods:
+
+- **OPTION 1** Create a cluster in WCS (for more details, refer to the [Weaviate Cluster Setup Guide](https://weaviate.io/developers/wcs/guides/create-instance))
+- **OPTION 2** Use Docker-Compose to setup a cluster locally [Weaviate Docker Guide](https://weaviate.io/developers/weaviate/installation/docker-compose)
+
+
+All connector functionality can be found in the [`connection.py`](./st_weaviate_connection/connection.py) python file. Documentation about `st.experimental_connection` can be found [here](https://docs.streamlit.io/library/api-reference/connections/st.experimental_connection).
+
+
+## ✨ Streamlit
+
+You can start the Streamlit app with the following command:
+
+```python
+streamlit run streamlit_app.py
+```
+
+## 📦 Data Management
+
+To use the demo locally, you need to import Magic card data into your Weaviate cluster. Inside the [data](./data/) directory, we provide three scripts.
+
+- `add_card_schema.py` Adds a card schema to your Weaviate cluster.
+- `delete_card_schema.py` Deletes the card schema and all saved objects.
+- `retrieve_magic_cards.py` Uses the [Scryfall API](https://scryfall.com/) to retrieve card information and saves them to your cluster.
+
+
+## 💖 Open Source Contribution
+
+Now, you're all set to use the Weaviate Connector for Streamlit. Happy coding!
+
+We encourage open-source contributions. Feel free to suggest improvements, provide feedback, create issues, and submit bug reports!

data/add_card_schema.py

@@ -0,0 +1,46 @@
+import weaviate  # type: ignore[import]
+import typer
+import os
+import json
+
+from wasabi import msg  # type: ignore[import]
+
+from dotenv import load_dotenv
+
+load_dotenv("../.env")
+
+
+def main() -> None:
+    msg.divider("Starting schema creation")
+
+    # Connect to Weaviate
+    url = os.environ.get("WEAVIATE_URL", "")
+    openai_key = os.environ.get("OPENAI_API_KEY", "")
+    auth_config = weaviate.AuthApiKey(api_key=os.environ.get("WEAVIATE_API_KEY", ""))
+
+    if openai_key == "" or url == "":
+        msg.fail("Environment Variables not set.")
+        msg.warn(f"URL: {url}")
+        msg.warn(f"OPENAI API KEY: {openai_key}")
+        return
+
+    client = weaviate.Client(
+        url=url,
+        additional_headers={"X-OpenAI-Api-Key": openai_key},
+        auth_client_secret=auth_config,
+    )
+
+    msg.good("Client connected to Weaviate Server")
+
+    if client.schema.exists("Card"):
+        msg.warn("Card class already exists")
+        return
+
+    with open("weaviate_schema.json", "r") as reader:
+        class_obj = json.load(reader)
+        client.schema.create_class(class_obj)
+        msg.good("Card class created")
+
+
+if __name__ == "__main__":
+    typer.run(main)