LLM/khoj
1
0
Fork 0
mirror of https://github.com/khoj-ai/khoj.git synced 2026-05-13 21:41:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: LLM:1.30.2
Branches Tags
LLM:master LLM:release/1.x LLM:v2/add-live-voice-chat LLM:debanjum/miscellaneous-v2-changes LLM:use-user-specified-chat-model-for-intermediate-steps LLM:pr/hjamet/1109 LLM:create-data-tracer LLM:evals/tmp-distill-trains-of-thought LLM:load-web-app-from-desktop-app LLM:use-structured-index-to-improve-kb-retrieval LLM:speed-up-khoj-docker-builds LLM:features/support-sqlite-as-db LLM:regenerate-edit-messages-on-web-app LLM:improve-agents-pane-on-home-screen LLM:use-turnstyle-ux-for-khoj-chat LLM:improve-doc-chunking-and-support-timely-embedings LLM:speed-up-embedding-model-app-load-when-not-connected-to-internet LLM:panchayat
LLM:2.0.0-beta.28 LLM:2.0.0-beta.27 LLM:2.0.0-beta.26 LLM:2.0.0-beta.25 LLM:2.0.0-beta.24 LLM:2.0.0-beta.23 LLM:2.0.0-beta.22 LLM:2.0.0-beta.21 LLM:2.0.0-beta.20 LLM:2.0.0-beta.19 LLM:2.0.0-beta.18 LLM:2.0.0-beta.17 LLM:2.0.0-beta.16 LLM:2.0.0-beta.15 LLM:2.0.0-beta.14 LLM:2.0.0-beta.13 LLM:2.0.0-beta.12 LLM:2.0.0-beta.11 LLM:2.0.0-beta.10 LLM:2.0.0-beta.9 LLM:2.0.0-beta.8 LLM:1.42.10 LLM:2.0.0-beta.7 LLM:2.0.0-beta.6 LLM:2.0.0-beta.5 LLM:2.0.0-beta.4 LLM:2.0.0-beta.3 LLM:2.0.0-beta.2 LLM:2.0.0-beta.1 LLM:1.42.9 LLM:1.42.8 LLM:1.42.7 LLM:1.42.6 LLM:1.42.5 LLM:1.42.4 LLM:1.42.3 LLM:1.42.2 LLM:1.42.1 LLM:1.42.0 LLM:1.41.0 LLM:1.40.0 LLM:1.39.0 LLM:1.38.0 LLM:1.37.2 LLM:1.37.1 LLM:1.37.0 LLM:1.36.6 LLM:1.36.5 LLM:1.36.4 LLM:1.36.3 LLM:1.36.2 LLM:1.36.1 LLM:1.36.0 LLM:1.35.3 LLM:1.35.2 LLM:1.35.1 LLM:1.35.0 LLM:1.34.0 LLM:1.33.2 LLM:1.33.1 LLM:1.33.0 LLM:1.32.2 LLM:1.32.1 LLM:1.32.0 LLM:1.31.0 LLM:1.30.10 LLM:1.30.9 LLM:1.30.8 LLM:1.30.7 LLM:1.30.6 LLM:1.30.5 LLM:1.30.4 LLM:1.30.3 LLM:1.30.2 LLM:1.30.1 LLM:1.30.0 LLM:1.29.1 LLM:1.29.0 LLM:1.28.3 LLM:1.28.2 LLM:1.28.1 LLM:1.28.0 LLM:1.27.1 LLM:1.27.0 LLM:1.26.4 LLM:1.26.3 LLM:1.26.2 LLM:1.26.1 LLM:1.26.0 LLM:1.25.0 LLM:1.24.1 LLM:1.24.0 LLM:1.23.3 LLM:1.23.2 LLM:1.23.1 LLM:1.23.0 LLM:1.22.3 LLM:1.22.2 LLM:1.22.1 LLM:1.22.0 LLM:1.21.6 LLM:1.21.5 LLM:1.21.4 LLM:1.21.3 LLM:1.21.2 LLM:1.21.1 LLM:1.21.0 LLM:1.20.4 LLM:1.20.3 LLM:1.20.2 LLM:1.20.1 LLM:1.20.0 LLM:1.17.0 LLM:1.16.0 LLM:1.15.0 LLM:1.14.0 LLM:1.13.0 LLM:1.12.1 LLM:1.12.0 LLM:1.11.2 LLM:1.11.1 LLM:1.11.0 LLM:1.10.2 LLM:1.10.1 LLM:1.10.0 LLM:1.9.0 LLM:1.8.0 LLM:1.7.0 LLM:1.6.2 LLM:1.6.1 LLM:1.6.0 LLM:1.5.1 LLM:1.5.0 LLM:1.4.0 LLM:1.3.0 LLM:1.2.1 LLM:1.2.0 LLM:1.1.0 LLM:1.0.1 LLM:1.0.0 LLM:0.14.0 LLM:0.13.0 LLM:0.12.3 LLM:0.12.2 LLM:0.12.1 LLM:rm LLM:0.12.0 LLM:0.11.4 LLM:0.11.3 LLM:0.11.2 LLM:0.11.1 LLM:0.11.0 LLM:0.10.1 LLM:0.10.0 LLM:0.9.0 LLM:0.8.2 LLM:0.8.1 LLM:0.8.0 LLM:0.7.1 LLM:0.7.0 LLM:0.6.2 LLM:0.6.1 LLM:0.6.0 LLM:0.5.0 LLM:0.4.0 LLM:0.3.0 LLM:0.2.6 LLM:0.2.5 LLM:0.2.1 LLM:v0.2.0 LLM:v0.1.9 LLM:v0.1.8 LLM:v0.1.7 LLM:v0.1.6 LLM:v0.1.5-alpha
...
compare: LLM:1.30.10
Branches Tags
LLM:master LLM:release/1.x LLM:v2/add-live-voice-chat LLM:debanjum/miscellaneous-v2-changes LLM:use-user-specified-chat-model-for-intermediate-steps LLM:pr/hjamet/1109 LLM:create-data-tracer LLM:evals/tmp-distill-trains-of-thought LLM:load-web-app-from-desktop-app LLM:use-structured-index-to-improve-kb-retrieval LLM:speed-up-khoj-docker-builds LLM:features/support-sqlite-as-db LLM:regenerate-edit-messages-on-web-app LLM:improve-agents-pane-on-home-screen LLM:use-turnstyle-ux-for-khoj-chat LLM:improve-doc-chunking-and-support-timely-embedings LLM:speed-up-embedding-model-app-load-when-not-connected-to-internet LLM:panchayat
LLM:2.0.0-beta.28 LLM:2.0.0-beta.27 LLM:2.0.0-beta.26 LLM:2.0.0-beta.25 LLM:2.0.0-beta.24 LLM:2.0.0-beta.23 LLM:2.0.0-beta.22 LLM:2.0.0-beta.21 LLM:2.0.0-beta.20 LLM:2.0.0-beta.19 LLM:2.0.0-beta.18 LLM:2.0.0-beta.17 LLM:2.0.0-beta.16 LLM:2.0.0-beta.15 LLM:2.0.0-beta.14 LLM:2.0.0-beta.13 LLM:2.0.0-beta.12 LLM:2.0.0-beta.11 LLM:2.0.0-beta.10 LLM:2.0.0-beta.9 LLM:2.0.0-beta.8 LLM:1.42.10 LLM:2.0.0-beta.7 LLM:2.0.0-beta.6 LLM:2.0.0-beta.5 LLM:2.0.0-beta.4 LLM:2.0.0-beta.3 LLM:2.0.0-beta.2 LLM:2.0.0-beta.1 LLM:1.42.9 LLM:1.42.8 LLM:1.42.7 LLM:1.42.6 LLM:1.42.5 LLM:1.42.4 LLM:1.42.3 LLM:1.42.2 LLM:1.42.1 LLM:1.42.0 LLM:1.41.0 LLM:1.40.0 LLM:1.39.0 LLM:1.38.0 LLM:1.37.2 LLM:1.37.1 LLM:1.37.0 LLM:1.36.6 LLM:1.36.5 LLM:1.36.4 LLM:1.36.3 LLM:1.36.2 LLM:1.36.1 LLM:1.36.0 LLM:1.35.3 LLM:1.35.2 LLM:1.35.1 LLM:1.35.0 LLM:1.34.0 LLM:1.33.2 LLM:1.33.1 LLM:1.33.0 LLM:1.32.2 LLM:1.32.1 LLM:1.32.0 LLM:1.31.0 LLM:1.30.10 LLM:1.30.9 LLM:1.30.8 LLM:1.30.7 LLM:1.30.6 LLM:1.30.5 LLM:1.30.4 LLM:1.30.3 LLM:1.30.2 LLM:1.30.1 LLM:1.30.0 LLM:1.29.1 LLM:1.29.0 LLM:1.28.3 LLM:1.28.2 LLM:1.28.1 LLM:1.28.0 LLM:1.27.1 LLM:1.27.0 LLM:1.26.4 LLM:1.26.3 LLM:1.26.2 LLM:1.26.1 LLM:1.26.0 LLM:1.25.0 LLM:1.24.1 LLM:1.24.0 LLM:1.23.3 LLM:1.23.2 LLM:1.23.1 LLM:1.23.0 LLM:1.22.3 LLM:1.22.2 LLM:1.22.1 LLM:1.22.0 LLM:1.21.6 LLM:1.21.5 LLM:1.21.4 LLM:1.21.3 LLM:1.21.2 LLM:1.21.1 LLM:1.21.0 LLM:1.20.4 LLM:1.20.3 LLM:1.20.2 LLM:1.20.1 LLM:1.20.0 LLM:1.17.0 LLM:1.16.0 LLM:1.15.0 LLM:1.14.0 LLM:1.13.0 LLM:1.12.1 LLM:1.12.0 LLM:1.11.2 LLM:1.11.1 LLM:1.11.0 LLM:1.10.2 LLM:1.10.1 LLM:1.10.0 LLM:1.9.0 LLM:1.8.0 LLM:1.7.0 LLM:1.6.2 LLM:1.6.1 LLM:1.6.0 LLM:1.5.1 LLM:1.5.0 LLM:1.4.0 LLM:1.3.0 LLM:1.2.1 LLM:1.2.0 LLM:1.1.0 LLM:1.0.1 LLM:1.0.0 LLM:0.14.0 LLM:0.13.0 LLM:0.12.3 LLM:0.12.2 LLM:0.12.1 LLM:rm LLM:0.12.0 LLM:0.11.4 LLM:0.11.3 LLM:0.11.2 LLM:0.11.1 LLM:0.11.0 LLM:0.10.1 LLM:0.10.0 LLM:0.9.0 LLM:0.8.2 LLM:0.8.1 LLM:0.8.0 LLM:0.7.1 LLM:0.7.0 LLM:0.6.2 LLM:0.6.1 LLM:0.6.0 LLM:0.5.0 LLM:0.4.0 LLM:0.3.0 LLM:0.2.6 LLM:0.2.5 LLM:0.2.1 LLM:v0.2.0 LLM:v0.1.9 LLM:v0.1.8 LLM:v0.1.7 LLM:v0.1.6 LLM:v0.1.5-alpha

42 Commits
1.30.2 ... 1.30.10

Author SHA1 Message Date
sabaimran
439b18c21f Release Khoj version 1.30.10 2024-11-28 19:43:06 -08:00
sabaimran
2dfd163430 Add more explicity run strategies in the runner matrix 2024-11-28 19:31:34 -08:00
sabaimran
80cd902c86 Since linux/amd64 images aren't being created, try setting a custom description on the image 
Refer to this GH documentation on working with multi arch images in the container registry:
https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#adding-a-description-to-multi-arch-images
 2024-11-28 19:14:06 -08:00
sabaimran
40d8a7a581 Release Khoj version 1.30.9 2024-11-28 18:45:50 -08:00
sabaimran
87aa653c7f Add additional steps in prod.Dockerfile to ensure dependencies are copied over 2024-11-28 18:37:08 -08:00
Debanjum
a552543f4f Use json5 to parse llm generated questions to query docs and web 
json5 is more forgiving, handles double quotes, newlines in raw json
string
 2024-11-28 14:35:34 -08:00
Debanjum
0a69af4f61 Update to latest ToDesktop runtime 2024-11-28 13:56:14 -08:00
Debanjum
1d0fe141dc Release Khoj version 1.30.8 2024-11-28 13:37:30 -08:00
Debanjum
29e801c381 Add MATH500 dataset to eval 
Evaluate simpler MATH500 responses with gemini 1.5 flash

This improves both the speed and cost of running this eval
 2024-11-28 12:48:25 -08:00
Debanjum
22aef9bf53 Add GPQA (diamond) dataset to eval 2024-11-28 12:48:25 -08:00
Debanjum
f1190ccf32 Improve parsing complex json strings returned by LLM (#989) 
- Improve escaping to load complex json objects
- Fallback to a more forgiving [json5](https://json5.org/) loader if `json.loads` cannot parse complex json str

This should reduce failures to pick research tool and run code by agent
 2024-11-28 11:01:39 -08:00
Debanjum
8c120a5139 Fallback to json5 loader if json.loads cannot parse complex json str 
JSON5 spec is more flexible, try to load using a fast json5 parser if
the stricter json.loads from the standard library can't load the
raw complex json string into a python dictionary/list
 2024-11-26 21:17:00 -08:00
Debanjum
70b7e7c73a Improve load of complex json objects. Use it to pick tool, run code 
Gemini doesn't work well when trying to output json objects. Using it
to output raw json strings with complex, multi-line structures
requires more intense clean-up of raw json string for parsing
 2024-11-26 17:37:57 -08:00
Debanjum
8cb0db0051 Fix llama-cpp-python install by pytest github workflow 
- Use pre-built wheels for torch and llama-cpp-python
- Install and link musl as it's used by llama-cpp-python pre-built
  wheel instead of glibc
- Join Install git and Install Dependencies steps in pytest workflow
  To remove unnecessary steps
 2024-11-26 02:04:36 -08:00
Debanjum
29315f44e7 Add assetlinks.json to link android app to app.khoj.dev domain 
Add sha cert of android upload, signing keys to open debug, prod apps
as TWA in fullscreen on android phones
 2024-11-26 01:57:54 -08:00
Debanjum
a97a45bf20 Align agent personality with recently updated khoj personality 
See update to Khoj personality in commit
6eb59464da
 2024-11-26 00:06:16 -08:00
Debanjum
e088fcbc7b Build for arm64 on arm64 runner. Parallelize arm64, x64 docker builds 
- Building arm64 image on an ubuntu arm64 runner reduces `yarn build'
  step time by 75% from 12mins to 3mins.
  - This is because no QEMU emulation for arm64 on x86 is required now
- Parallelizing x64 and arm64 platform builds halves build time on top
  - Revert to use standard ubuntu-latest runner as large x64 runner
    doesn't give much more speed improvements

This results an effective additional 50%-66% reduction in build time
on top of #987.

So a full dockerize workflow run now takes *10 mins* vs previous 35+mins.
This is a total of *72% improvement* in max dockerize run time.

Get additional speed improvements when docker layer cache hit.
 2024-11-24 23:18:55 -08:00
Debanjum
5723a3778e Speed up Docker image builds using multi-stage parallel pipelines (#987) 
## Objective
Improve build speed and size of khoj docker images

## Changes
### Improve docker image build speeds
  - Decouple web app and server build steps
  - Build the web app and server in parallel
  - Cache docker layers for reuse across dockerize github workflow runs
    - Split Docker build layers for improved cacheability (e.g separate `yarn install` and `yarn build` steps)
### Reduce size of khoj docker images 
  - Use an up-to-date `.dockerignore` to exclude unnecessary directories
  - Do not installing cuda python packages for cpu builds
### Improve web app builds
  - Use consistent mechanism to get fonts for web app
  - Make tailwind extensions production instead of dev dependencies
  - Make next.js create production builds for the web app (via `NODE_ENV=production` env var)
 2024-11-24 21:49:46 -08:00
Debanjum
4a5646c8da Cache docker layers, nextjs builds in dockerize github workflow 2024-11-24 21:06:22 -08:00
Debanjum
6a39651ad3 Standardize loading fonts locally across pages on web app 2024-11-24 20:41:15 -08:00
sabaimran
9368699b2c Migrate the pre-commit config 2024-11-24 14:54:26 -08:00
sabaimran
6eb59464da Add additional reinforcement to coax gemini into giving a minimum helpful response 2024-11-24 14:53:53 -08:00
sabaimran
15f062b34a Remove print statement for agent style map 2024-11-24 14:53:53 -08:00
sabaimran
d7e68a2d1b Wait for iplcodata to load before first message 
- Fix the console khoj ai ascii art
- Remove some not so good suggested prompt
 2024-11-24 14:53:53 -08:00
Debanjum
f51e0f7859 Make Next.js create production builds of web app for Docker images 2024-11-24 13:59:40 -08:00
Debanjum
710e00ad9e Make tailwind extensions prod, instead of dev, deps of web app 2024-11-24 13:59:40 -08:00
Debanjum
4b486ea5f6 Exclude unnecessary directories from final docker builds 2024-11-24 13:59:40 -08:00
Debanjum
78d8ca49ec Skip Nvidia GPU python packages during Server install in Dockerfiles 2024-11-24 13:59:39 -08:00
Debanjum
37887a175a Speed up Docker image builds using multi-stage parallel pipelines 
Decouple web app, server builds in parallel to speed up Docker
builds
 2024-11-24 12:48:30 -08:00
Debanjum
7c77d65d35 Improve logic to disable telemetry via KHOJ_TELEMETRY_DISABLE env var 
The newly added KHOJ_TELEMETRY_DISABLE env var knob to disable
telemetry should override old config mechanism when set
 2024-11-24 00:54:16 -08:00
sabaimran
2d683898c2 Release Khoj version 1.30.7 2024-11-23 22:51:10 -08:00
sabaimran
914ff994f7 Fix cost addition to chat_metadata 2024-11-23 22:50:45 -08:00
Debanjum
caaa127dcf Release Khoj version 1.30.6 2024-11-23 21:07:00 -08:00
Debanjum
57b8273002 Fix apt install for musl-dev in prod.Dockerfile 2024-11-23 21:06:09 -08:00
Debanjum
8f966b11ec Release Khoj version 1.30.5 2024-11-23 20:49:05 -08:00
Debanjum
498895a47d Fix libmusl error using pre-built llama-cpp-python wheel in prod Docker 2024-11-23 20:47:41 -08:00
Debanjum
e5b211a743 Release Khoj version 1.30.4 2024-11-23 19:48:21 -08:00
Debanjum
9848d89d03 Try build docker images with github high cpu, ram runner 2024-11-23 19:09:36 -08:00
Debanjum
04bb3d6f15 Fix libmusl error using pre-built llama-cpp-python wheel via Docker 
Seems like llama-cpp-python pre-built wheels need libmusl. Otherwise
you run into runtime errors on Khoj startup via Docker.
 2024-11-23 18:46:44 -08:00
Debanjum
8dd2122817 Set sample size to 200 for automated eval runs as well 2024-11-23 14:48:38 -08:00
Debanjum
c4ef31d86f Release Khoj version 1.30.3 2024-11-23 14:40:06 -08:00
Debanjum
15ae22bdcf Use pre-built llama-cpp-python wheel in Khoj docker images 
Reduces build time and resolves FileNotFoundError 'ninja' during
llama-cpp-python local build.
 2024-11-23 14:38:07 -08:00
46 changed files with 496 additions and 220 deletions
Expand all files Collapse all files

13
.dockerignore
View File

@@ -1,10 +1,11 @@
.git/
.pytest_cache/
.vscode/
.venv/
docs/
.*
**/__pycache__/
*.egg-info/
documentation/
tests/
build/
dist/
scripts/
*.egg-info/
src/interface/
src/telemetry/
!src/interface/web

36
.github/workflows/dockerize.yml vendored
View File

@@ -38,13 +38,23 @@ env:
jobs:
build:
name: Publish Khoj Docker Images
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
image:
- 'local'
- 'cloud'
include:
- image: 'local'
platform: linux/amd64
runner: ubuntu-latest
- image: 'local'
platform: linux/arm64
runner: ubuntu-linux-arm64
- image: 'cloud'
platform: linux/amd64
runner: ubuntu-latest
- image: 'cloud'
platform: linux/arm64
runner: ubuntu-linux-arm64
runs-on: ${{ matrix.runner }}
steps:
- name: Checkout Code
uses: actions/checkout@v3
@@ -73,12 +83,12 @@ jobs:
run: rm -rf /opt/hostedtoolcache
- name: 📦 Build and Push Docker Image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v4
if: (matrix.image == 'local' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj == 'true' || (matrix.image == 'local' && github.event_name == 'push')
with:
context: .
file: Dockerfile
platforms: linux/amd64, linux/arm64
platforms: ${{ matrix.platform }}
push: true
tags: |
ghcr.io/${{ github.repository }}:${{ env.DOCKER_IMAGE_TAG }}
@@ -86,14 +96,19 @@ jobs:
build-args: |
VERSION=${{ steps.hatch.outputs.version }}
PORT=42110
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.platform }}
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.platform}}
labels: |
org.opencontainers.image.description=Khoj AI - Your second brain powered by LLMs and Neural Search
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
- name: 📦️⛅️ Build and Push Cloud Docker Image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v4
if: (matrix.image == 'cloud' && github.event_name == 'workflow_dispatch') && github.event.inputs.khoj-cloud == 'true' || (matrix.image == 'cloud' && github.event_name == 'push')
with:
context: .
file: prod.Dockerfile
platforms: linux/amd64, linux/arm64
platforms: ${{ matrix.platform }}
push: true
tags: |
ghcr.io/${{ github.repository }}-cloud:${{ env.DOCKER_IMAGE_TAG }}
@@ -101,3 +116,8 @@ jobs:
build-args: |
VERSION=${{ steps.hatch.outputs.version }}
PORT=42110
cache-from: type=gha,scope=${{ matrix.image }}-${{ matrix.platform }}
cache-to: type=gha,mode=max,scope=${{ matrix.image }}-${{ matrix.platform}}
labels: |
org.opencontainers.image.description=Khoj AI Cloud - Your second brain powered by LLMs and Neural Search
org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}

9
.github/workflows/run_evals.yml vendored
View File

@@ -25,6 +25,8 @@ on:
options:
- frames
- simpleqa
- gpqa
- math500
sample_size:
description: 'Number of samples to evaluate'
required: false
@@ -89,14 +91,15 @@ jobs:
- name: 📝 Run Eval
env:
KHOJ_MODE: ${{ matrix.khoj_mode }}
SAMPLE_SIZE: ${{ inputs.sample_size }}
SAMPLE_SIZE: ${{ github.event_name == 'workflow_dispatch' && inputs.sample_size || 200 }}
BATCH_SIZE: "20"
RANDOMIZE: "True"
KHOJ_URL: "http://localhost:42110"
KHOJ_LLM_SEED: "42"
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
SERPER_DEV_API_KEY: ${{ secrets.SERPER_DEV_API_KEY }}
OLOSTEP_API_KEY: ${{ secrets.OLOSTEP_API_KEY }}
SERPER_DEV_API_KEY: ${{ matrix.dataset != 'math500' && secrets.SERPER_DEV_API_KEY }}
OLOSTEP_API_KEY: ${{ matrix.dataset != 'math500' && secrets.OLOSTEP_API_KEY }}
HF_TOKEN: ${{ secrets.HF_TOKEN }}
KHOJ_ADMIN_EMAIL: khoj
KHOJ_ADMIN_PASSWORD: khoj
POSTGRES_HOST: localhost

11
.github/workflows/test.yml vendored
View File

@@ -55,15 +55,13 @@ jobs:
with:
python-version: ${{ matrix.python_version }}
- name: Install Git
run: |
apt update && apt install -y git
- name: ⏬️ Install Dependencies
env:
DEBIAN_FRONTEND: noninteractive
run: |
apt update && apt install -y libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
apt update && apt install -y git libegl1 sqlite3 libsqlite3-dev libsqlite3-0 ffmpeg libsm6 libxext6
# required by llama-cpp-python prebuilt wheels
apt install -y musl-dev && ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1
- name: ⬇️ Install Postgres
env:
@@ -78,6 +76,9 @@ jobs:
python -m pip install --upgrade pip
- name: ⬇️ Install Application
env:
PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
CUDA_VISIBLE_DEVICES: ""
run: sed -i 's/dynamic = \["version"\]/version = "0.0.0"/' pyproject.toml && pip install --upgrade .[dev]
- name: 🧪 Test Application

2
.pre-commit-config.yaml
View File

@@ -26,7 +26,7 @@ repos:
rev: v1.0.0
hooks:
- id: mypy
stages: [push, manual]
stages: [pre-push, manual]
pass_filenames: false
args:
- --config-file=pyproject.toml

49
Dockerfile
View File

@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
FROM ubuntu:jammy
FROM ubuntu:jammy AS base
LABEL homepage="https://khoj.dev"
LABEL repository="https://github.com/khoj-ai/khoj"
LABEL org.opencontainers.image.source="https://github.com/khoj-ai/khoj"
@@ -13,36 +13,51 @@ RUN apt update -y && apt -y install \
# Required by RapidOCR
libgl1 \
libglx-mesa0 \
libglib2.0-0 && \
# Required by Next.js Web app
curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
apt update -y && apt -y --no-install-recommends install nodejs yarn && \
libglib2.0-0 \
# Required by llama-cpp-python pre-built wheels. See #1628
musl-dev && \
ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 && \
# Clean up
apt clean && rm -rf /var/lib/apt/lists/*
# Install Application
# Build Server
FROM base AS server-deps
WORKDIR /app
COPY pyproject.toml .
COPY README.md .
ARG VERSION=0.0.0
# use the pre-built llama-cpp-python, torch cpu wheel
ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
# avoid downloading unused cuda specific python packages
ENV CUDA_VISIBLE_DEVICES=""
RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
pip install --no-cache-dir .
# Copy Source Code
COPY . .
# Set the PYTHONPATH environment variable in order for it to find the Django app.
ENV PYTHONPATH=/app/src:$PYTHONPATH
# Go to the directory src/interface/web and export the built Next.js assets
# Build Web App
FROM node:20-alpine AS web-app
# Set build optimization env vars
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
RUN bash -c "yarn install --frozen-lockfile && yarn ciexport && yarn cache clean"
# Install dependencies first (cache layer)
COPY src/interface/web/package.json src/interface/web/yarn.lock ./
RUN yarn install --frozen-lockfile
# Copy source and build
COPY src/interface/web/. ./
RUN yarn build
# Merge the Server and Web App into a Single Image
FROM base
ENV PYTHONPATH=/app/src
WORKDIR /app
COPY --from=server-deps /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
COPY --from=web-app /app/src/interface/web/out ./src/khoj/interface/built
COPY . .
RUN cd src && python3 khoj/manage.py collectstatic --noinput
# Run the Application
# There are more arguments required for the application to run,
# but these should be passed in through the docker-compose.yml file.
# but those should be passed in through the docker-compose.yml file.
ARG PORT
EXPOSE ${PORT}
ENTRYPOINT ["python3", "src/khoj/main.py"]

2
manifest.json
View File

@@ -1,7 +1,7 @@
{
"id": "khoj",
"name": "Khoj",
"version": "1.30.2",
"version": "1.30.10",
"minAppVersion": "0.15.0",
"description": "Your Second Brain",
"author": "Khoj Inc.",

50
prod.Dockerfile
View File

@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
FROM ubuntu:jammy
FROM ubuntu:jammy AS base
LABEL homepage="https://khoj.dev"
LABEL repository="https://github.com/khoj-ai/khoj"
LABEL org.opencontainers.image.source="https://github.com/khoj-ai/khoj"
@@ -13,36 +13,52 @@ RUN apt update -y && apt -y install \
libsm6 \
libxext6 \
swig \
curl && \
# Required by Next.js Web app
curl -sL https://deb.nodesource.com/setup_20.x | bash - && \
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list && \
apt update -y && apt -y --no-install-recommends install nodejs yarn && \
curl \
# Required by llama-cpp-python pre-built wheels. See #1628
musl-dev && \
ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 && \
# Clean up
apt clean && rm -rf /var/lib/apt/lists/*
# Install Application
# Build Server
FROM base AS server-deps
WORKDIR /app
COPY pyproject.toml .
COPY README.md .
ARG VERSION=0.0.0
# use the pre-built llama-cpp-python, torch cpu wheel
ENV PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu https://abetlen.github.io/llama-cpp-python/whl/cpu"
# avoid downloading unused cuda specific python packages
ENV CUDA_VISIBLE_DEVICES=""
RUN sed -i "s/dynamic = \\[\"version\"\\]/version = \"$VERSION\"/" pyproject.toml && \
pip install --no-cache-dir -e .[prod]
# Copy Source Code
COPY . .
# Set the PYTHONPATH environment variable in order for it to find the Django app.
ENV PYTHONPATH=/app/src:$PYTHONPATH
# Go to the directory src/interface/web and export the built Next.js assets
# Build Web App
FROM node:20-alpine AS web-app
# Set build optimization env vars
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
WORKDIR /app/src/interface/web
RUN bash -c "yarn install --frozen-lockfile --verbose && yarn ciexport && yarn cache clean"
# Install dependencies first (cache layer)
COPY src/interface/web/package.json src/interface/web/yarn.lock ./
RUN yarn install --frozen-lockfile
# Copy source and build
COPY src/interface/web/. ./
RUN yarn build
# Merge the Server and Web App into a Single Image
FROM base
ENV PYTHONPATH=/app/src:$PYTHONPATH
WORKDIR /app
COPY --from=server-deps /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages
COPY --from=server-deps /usr/local/bin /usr/local/bin
COPY --from=web-app /app/src/interface/web/out ./src/khoj/interface/built
COPY . .
RUN cd src && python3 khoj/manage.py collectstatic --noinput
# Run the Application
# There are more arguments required for the application to run,
# but these should be passed in through the docker-compose.yml file.
# but those should be passed in through the docker-compose.yml file.
ARG PORT
EXPOSE ${PORT}
ENTRYPOINT ["gunicorn", "-c", "gunicorn-config.py", "src.khoj.main:app"]

1
pyproject.toml
View File

@@ -88,6 +88,7 @@ dependencies = [
"anthropic == 0.26.1",
"docx2txt == 0.8",
"google-generativeai == 0.8.3",
"pyjson5 == 1.6.7",
]
dynamic = ["version"]

4
src/interface/desktop/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.30.2",
"version": "1.30.10",
"description": "Your Second Brain",
"author": "Khoj Inc. <team@khoj.dev>",
"license": "GPL-3.0-or-later",
@@ -16,7 +16,7 @@
"start": "yarn electron ."
},
"dependencies": {
"@todesktop/runtime": "^1.6.4",
"@todesktop/runtime": "^2.0.0",
"axios": "^1.7.4",
"cron": "^2.4.3",
"electron-store": "^8.1.0"

90
src/interface/desktop/yarn.lock
View File

@@ -50,17 +50,17 @@
dependencies:
defer-to-connect "^2.0.0"
"@todesktop/runtime@^1.6.4":
version "1.6.4"
resolved "https://registry.yarnpkg.com/@todesktop/runtime/-/runtime-1.6.4.tgz#a9d62a021cf2647c51371c892bfb1d4c5a29ed7e"
integrity sha512-n6dOxhrKKsXMM+i2u9iRvoJSR2KCWw0orYK+FT9RbWNPykhuFIYd0yy8dYgYy/OuClKGyGl4SJFi2757FLhWDA==
"@todesktop/runtime@^2.0.0":
version "2.0.0"
resolved "https://registry.yarnpkg.com/@todesktop/runtime/-/runtime-2.0.0.tgz#dfd409186ae664f5e28186a03b99e620ec7b7f82"
integrity sha512-0a2tmWpIc/HJE/873xRMZKQNggfrYhoKYIchfN+k8RqKdzTPwTWa5ztur7GdCHLHBUaiMBPNRzF3h4kwHd1NCw==
dependencies:
del "^6.0.0"
electron-updater "^4.6.1"
eventemitter2 "^6.4.5"
del "^6.1.1"
electron-updater "^6.3.9"
eventemitter2 "^6.4.9"
execa "^5.0.0"
lodash.once "^4.1.1"
semver "^7.3.2"
semver "^7.6.3"
"@types/cacheable-request@^6.0.1":
version "6.0.3"
@@ -90,16 +90,16 @@
integrity sha512-jYvz8UMLDgy3a5SkGJne8H7VA7zPV2Lwohjx0V8V31+SqAjNmurWMkk9cQhfvlcnXWudBpK9xPM1n4rljOcHYQ==
"@types/node@*":
version "22.9.1"
resolved "https://registry.yarnpkg.com/@types/node/-/node-22.9.1.tgz#bdf91c36e0e7ecfb7257b2d75bf1b206b308ca71"
integrity sha512-p8Yy/8sw1caA8CdRIQBG5tiLHmxtQKObCijiAa9Ez+d4+PRffM4054xbju0msf+cvhJpnFEeNjxmVT/0ipktrg==
version "22.10.1"
resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.1.tgz#41ffeee127b8975a05f8c4f83fb89bcb2987d766"
integrity sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==
dependencies:
undici-types "~6.19.8"
undici-types "~6.20.0"
"@types/node@^18.11.18":
version "18.19.64"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.64.tgz#122897fb79f2a9ec9c979bded01c11461b2b1478"
integrity sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==
version "18.19.67"
resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.67.tgz#77c4b01641a1e3e1509aff7e10d39e4afd5ae06d"
integrity sha512-wI8uHusga+0ZugNp0Ol/3BqQfEcCCNfojtO6Oou9iVNGPTL6QNSdnUdqq85fRgIorLhLMuPIKpsN98QE9Nh+KQ==
dependencies:
undici-types "~5.26.4"
@@ -110,11 +110,6 @@
dependencies:
"@types/node" "*"
"@types/semver@^7.3.6":
version "7.5.8"
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.8.tgz#8268a8c57a3e4abd25c165ecd36237db7948a55e"
integrity sha512-I8EUhyrgfLrcTkzV3TSsGyl1tSuPrEDzr0yd5m90UgNxQkyDXULk3b6MlQqTCpZpNtWe1K0hzclnZkTcLBe2UQ==
"@types/yauzl@^2.9.1":
version "2.10.3"
resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999"
@@ -168,9 +163,9 @@ atomically@^1.7.0:
integrity sha512-Xcz9l0z7y9yQ9rdDaxlmaI4uJHf/T8g9hOEzJcsEqX2SjCj4J20uK7+ldkDHMbpJDK76wF7xEIgxc/vSlsfw5w==
axios@^1.7.4:
version "1.7.7"
resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f"
integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==
version "1.7.8"
resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.8.tgz#1997b1496b394c21953e68c14aaa51b7b5de3d6e"
integrity sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==
dependencies:
follow-redirects "^1.15.6"
form-data "^4.0.0"
@@ -206,12 +201,12 @@ buffer-crc32@~0.2.3:
resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242"
integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==
builder-util-runtime@8.9.2:
version "8.9.2"
resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-8.9.2.tgz#a9669ae5b5dcabfe411ded26678e7ae997246c28"
integrity sha512-rhuKm5vh7E0aAmT6i8aoSfEjxzdYEFX7zDApK+eNgOhjofnWb74d9SRJv0H/8nsgOkos0TZ4zxW0P8J4N7xQ2A==
builder-util-runtime@9.2.10:
version "9.2.10"
resolved "https://registry.yarnpkg.com/builder-util-runtime/-/builder-util-runtime-9.2.10.tgz#a0f7d9e214158402e78b74a745c8d9f870c604bc"
integrity sha512-6p/gfG1RJSQeIbz8TK5aPNkoztgY1q5TgmGFMAXcY8itsGW6Y2ld1ALsZ5UJn8rog7hKF3zHx5iQbNQ8uLcRlw==
dependencies:
debug "^4.3.2"
debug "^4.3.4"
sax "^1.2.4"
cacheable-lookup@^5.0.3:
@@ -296,7 +291,7 @@ debounce-fn@^4.0.0:
dependencies:
mimic-fn "^3.0.0"
debug@^4.1.0, debug@^4.1.1, debug@^4.3.2:
debug@^4.1.0, debug@^4.1.1, debug@^4.3.4:
version "4.3.7"
resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.7.tgz#87945b4151a011d76d95a198d7111c865c360a52"
integrity sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==
@@ -333,7 +328,7 @@ define-properties@^1.2.1:
has-property-descriptors "^1.0.0"
object-keys "^1.1.1"
del@^6.0.0:
del@^6.1.1:
version "6.1.1"
resolved "https://registry.yarnpkg.com/del/-/del-6.1.1.tgz#3b70314f1ec0aa325c6b14eb36b95786671edb7a"
integrity sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==
@@ -379,19 +374,19 @@ electron-store@^8.1.0:
conf "^10.2.0"
type-fest "^2.17.0"
electron-updater@^4.6.1:
version "4.6.5"
resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-4.6.5.tgz#e9a75458bbfd6bb41a58a829839e150ad2eb2d3d"
integrity sha512-kdTly8O9mSZfm9fslc1mnCY+mYOeaYRy7ERa2Fed240u01BKll3aiupzkd07qKw69KvhBSzuHroIW3mF0D8DWA==
electron-updater@^6.3.9:
version "6.3.9"
resolved "https://registry.yarnpkg.com/electron-updater/-/electron-updater-6.3.9.tgz#e1e7f155624c58e6f3760f376c3a584028165ec4"
integrity sha512-2PJNONi+iBidkoC5D1nzT9XqsE8Q1X28Fn6xRQhO3YX8qRRyJ3mkV4F1aQsuRnYPqq6Hw+E51y27W75WgDoofw==
dependencies:
"@types/semver" "^7.3.6"
builder-util-runtime "8.9.2"
fs-extra "^10.0.0"
builder-util-runtime "9.2.10"
fs-extra "^10.1.0"
js-yaml "^4.1.0"
lazy-val "^1.0.5"
lodash.escaperegexp "^4.1.2"
lodash.isequal "^4.5.0"
semver "^7.3.5"
semver "^7.6.3"
tiny-typed-emitter "^2.1.0"
electron@28.2.1:
version "28.2.1"
@@ -436,7 +431,7 @@ escape-string-regexp@^4.0.0:
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34"
integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==
eventemitter2@^6.4.5:
eventemitter2@^6.4.9:
version "6.4.9"
resolved "https://registry.yarnpkg.com/eventemitter2/-/eventemitter2-6.4.9.tgz#41f2750781b4230ed58827bc119d293471ecb125"
integrity sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==
@@ -530,7 +525,7 @@ form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
fs-extra@^10.0.0:
fs-extra@^10.1.0:
version "10.1.0"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf"
integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==
@@ -1115,7 +1110,7 @@ semver@^6.2.0:
resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==
semver@^7.3.2, semver@^7.3.5:
semver@^7.3.2, semver@^7.3.5, semver@^7.6.3:
version "7.6.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==
@@ -1166,6 +1161,11 @@ sumchecker@^3.0.1:
dependencies:
debug "^4.1.0"
tiny-typed-emitter@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/tiny-typed-emitter/-/tiny-typed-emitter-2.1.0.tgz#b3b027fdd389ff81a152c8e847ee2f5be9fad7b5"
integrity sha512-qVtvMxeXbVej0cQWKqVSSAHmKZEHAvxdF8HEUBFWts8h+xEo5m/lEiPakuyZ3BnCBjOD8i24kzNOiOLLgsSxhA==
to-regex-range@^5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4"
@@ -1188,10 +1188,10 @@ undici-types@~5.26.4:
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
undici-types@~6.19.8:
version "6.19.8"
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02"
integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==
undici-types@~6.20.0:
version "6.20.0"
resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433"
integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==
universalify@^0.1.0:
version "0.1.2"

2
src/interface/emacs/khoj.el
View File

@@ -6,7 +6,7 @@
;; Saba Imran <saba@khoj.dev>
;; Description: Your Second Brain
;; Keywords: search, chat, ai, org-mode, outlines, markdown, pdf, image
;; Version: 1.30.2
;; Version: 1.30.10
;; Package-Requires: ((emacs "27.1") (transient "0.3.0") (dash "2.19.1"))
;; URL: https://github.com/khoj-ai/khoj/tree/master/src/interface/emacs

2
src/interface/obsidian/manifest.json
View File

@@ -1,7 +1,7 @@
{
"id": "khoj",
"name": "Khoj",
"version": "1.30.2",
"version": "1.30.10",
"minAppVersion": "0.15.0",
"description": "Your Second Brain",
"author": "Khoj Inc.",

2
src/interface/obsidian/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "Khoj",
"version": "1.30.2",
"version": "1.30.10",
"description": "Your Second Brain",
"author": "Debanjum Singh Solanky, Saba Imran <team@khoj.dev>",
"license": "GPL-3.0-or-later",

10
src/interface/obsidian/versions.json
View File

@@ -93,5 +93,13 @@
"1.29.1": "0.15.0",
"1.30.0": "0.15.0",
"1.30.1": "0.15.0",
"1.30.2": "0.15.0"
"1.30.2": "0.15.0",
"1.30.3": "0.15.0",
"1.30.4": "0.15.0",
"1.30.5": "0.15.0",
"1.30.6": "0.15.0",
"1.30.7": "0.15.0",
"1.30.8": "0.15.0",
"1.30.9": "0.15.0",
"1.30.10": "0.15.0"
}

8
src/interface/web/app/agents/layout.tsx
View File

@@ -1,9 +1,7 @@
import type { Metadata } from "next";
import { Noto_Sans } from "next/font/google";
import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../globals.css";
const inter = Noto_Sans({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Khoj AI - Agents",
description: "Find a specialized agent that can help you address more specific needs.",
@@ -33,7 +31,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
<html lang="en">
<html lang="en" className={`${noto_sans.variable} ${noto_sans_arabic.variable}`}>
<meta
httpEquiv="Content-Security-Policy"
content="default-src 'self' https://assets.khoj.dev;
@@ -46,7 +44,7 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body>{children}</body>
</html>
);
}

24
src/interface/web/app/automations/page.tsx
View File

@@ -994,7 +994,7 @@ export default function Automations() {
const [suggestedAutomations, setSuggestedAutomations] = useState<AutomationsData[]>([]);
const [showLoginPrompt, setShowLoginPrompt] = useState(false);
const isMobileWidth = useIsMobileWidth();
const ipLocationData = useIPLocationData();
const { locationData, locationDataError, locationDataLoading } = useIPLocationData();
useEffect(() => {
if (newAutomationData) {
@@ -1044,18 +1044,18 @@ export default function Automations() {
{authenticatedData.email}
</span>
) : null}
{ipLocationData && (
{locationData && (
<span className="rounded-lg text-sm border-secondary border p-1 flex items-center shadow-sm">
<MapPinSimple className="h-4 w-4 mr-2 inline text-purple-500" />
{ipLocationData
? `${ipLocationData.city}, ${ipLocationData.country}`
{locationData
? `${locationData.city}, ${locationData.country}`
: "Unknown"}
</span>
)}
{ipLocationData && (
{locationData && (
<span className="rounded-lg text-sm border-secondary border p-1 flex items-center shadow-sm">
<Clock className="h-4 w-4 mr-2 inline text-green-500" />
{ipLocationData ? `${ipLocationData.timezone}` : "Unknown"}
{locationData ? `${locationData.timezone}` : "Unknown"}
</span>
)}
</div>
@@ -1086,7 +1086,7 @@ export default function Automations() {
setNewAutomationData={setNewAutomationData}
authenticatedData={authenticatedData}
isCreating={isCreating}
ipLocationData={ipLocationData}
ipLocationData={locationData}
/>
) : (
<Button
@@ -1103,7 +1103,7 @@ export default function Automations() {
<SharedAutomationCard
isMobileWidth={isMobileWidth}
authenticatedData={authenticatedData}
locationData={ipLocationData}
locationData={locationData}
isLoggedIn={authenticatedData ? true : false}
setShowLoginPrompt={setShowLoginPrompt}
setNewAutomationData={setNewAutomationData}
@@ -1125,7 +1125,7 @@ export default function Automations() {
setNewAutomationData={setNewAutomationData}
authenticatedData={authenticatedData}
isCreating={isCreating}
ipLocationData={ipLocationData}
ipLocationData={locationData}
/>
) : (
<Button
@@ -1147,7 +1147,7 @@ export default function Automations() {
key={automation.id}
authenticatedData={authenticatedData}
automation={automation}
locationData={ipLocationData}
locationData={locationData}
isLoggedIn={authenticatedData ? true : false}
setShowLoginPrompt={setShowLoginPrompt}
/>
@@ -1158,7 +1158,7 @@ export default function Automations() {
key={automation.id}
authenticatedData={authenticatedData}
automation={automation}
locationData={ipLocationData}
locationData={locationData}
isLoggedIn={authenticatedData ? true : false}
setShowLoginPrompt={setShowLoginPrompt}
/>
@@ -1173,7 +1173,7 @@ export default function Automations() {
key={automation.id}
authenticatedData={authenticatedData}
automation={automation}
locationData={ipLocationData}
locationData={locationData}
isLoggedIn={authenticatedData ? true : false}
setShowLoginPrompt={setShowLoginPrompt}
suggestedCard={true}

8
src/interface/web/app/chat/layout.tsx
View File

@@ -1,9 +1,7 @@
import type { Metadata } from "next";
import { Noto_Sans } from "next/font/google";
import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../globals.css";
const inter = Noto_Sans({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Khoj AI - Chat",
description:
@@ -34,7 +32,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
<html lang="en">
<html lang="en" className={`${noto_sans.variable} ${noto_sans_arabic.variable}`}>
<meta
httpEquiv="Content-Security-Policy"
content="default-src 'self' https://assets.khoj.dev;
@@ -47,7 +45,7 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>
<body>
{children}
<script
dangerouslySetInnerHTML={{

12
src/interface/web/app/chat/page.tsx
View File

@@ -184,8 +184,10 @@ export default function Chat() {
useState<AbortController | null>(null);
const [triggeredAbort, setTriggeredAbort] = useState(false);
const locationData = useIPLocationData() || {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
const { locationData, locationDataError, locationDataLoading } = useIPLocationData() || {
locationData: {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
},
};
const authenticatedData = useAuthenticatedData();
const isMobileWidth = useIsMobileWidth();
@@ -239,9 +241,13 @@ export default function Chat() {
useEffect(() => {
if (processQuerySignal) {
if (locationDataLoading) {
return;
}
chat();
}
}, [processQuerySignal]);
}, [processQuerySignal, locationDataLoading]);
async function readChatStream(response: Response) {
if (!response.ok) throw new Error(response.statusText);

34
src/interface/web/app/common/utils.ts
View File

@@ -23,32 +23,30 @@ export function welcomeConsole() {
`%c %s`,
"font-family:monospace",
`
__ __ __ __ ______ __ _____ __
/\\ \\/ / /\\ \\_\\ \\ /\\ __ \\ /\\ \\ /\\ __ \\ /\\ \\
\\ \\ _"-. \\ \\ __ \\ \\ \\ \\/\\ \\ _\\_\\ \\ \\ \\ __ \\ \\ \\ \\
\\ \\_\\ \\_\\ \\ \\_\\ \\_\\ \\ \\_____\\ /\\_____\\ \\ \\_\\ \\_\\ \\ \\_\\
\\/_/\\/_/ \\/_/\\/_/ \\/_____/ \\/_____/ \\/_/\\/_/ \\/_/
__ __ __ __ ______ __ _____ __
/\\ \\/ / /\\ \\_\\ \\ /\\ __ \\ /\\ \\ /\\ __ \\ /\\ \\
\\ \\ _"-. \\ \\ __ \\ \\ \\ \\/\\ \\ _\\_\\ \\ \\ \\ __ \\ \\ \\ \\
\\ \\_\\ \\_\\ \\ \\_\\ \\_\\ \\ \\_____\\ /\\_____\\ \\ \\_\\ \\_\\ \\ \\_\\
\\/_/\\/_/ \\/_/\\/_/ \\/_____/ \\/_____/ \\/_/\\/_/ \\/_/
Greetings traveller,
Greetings traveller,
I am ✨Khoj✨, your open-source, personal AI copilot.
I am ✨Khoj✨, your open-source, personal AI copilot.
See my source code at https://github.com/khoj-ai/khoj
Read my operating manual at https://docs.khoj.dev
`,
See my source code at https://github.com/khoj-ai/khoj
Read my operating manual at https://docs.khoj.dev
`,
);
}
export function useIPLocationData() {
const { data: locationData, error: locationDataError } = useSWR<LocationData>(
"/api/ip",
locationFetcher,
{ revalidateOnFocus: false },
);
if (locationDataError || !locationData) return;
return locationData;
const {
data: locationData,
error: locationDataError,
isLoading: locationDataLoading,
} = useSWR<LocationData>("/api/ip", locationFetcher, { revalidateOnFocus: false });
return { locationData, locationDataError, locationDataLoading };
}
export function useIsMobileWidth() {

1
src/interface/web/app/components/sidePanel/chatHistorySidePanel.tsx
View File

@@ -726,7 +726,6 @@ function ChatSessionsModal({ data, showSidePanel }: ChatSessionsModalProps) {
}
});
});
console.log(agentNameToStyleMapLocal);
setAgentNameToStyleMap(agentNameToStyleMapLocal);
setAgentOptions(agents);
}

6
src/interface/web/app/components/suggestions/suggestionsData.ts
View File

@@ -724,12 +724,6 @@ export const suggestionsData: Suggestion[] = [
description: "Draw a diagram illustrating the structure of the United States government.",
link: "",
},
{
type: SuggestionType.Health,
color: suggestionToColorMap[SuggestionType.Health] || DEFAULT_COLOR,
description: "Draw a diagram of the human skeletal system.",
link: "",
},
{
type: SuggestionType.Code,
color: suggestionToColorMap[SuggestionType.Code] || DEFAULT_COLOR,

13
src/interface/web/app/fonts.ts Normal file
View File

@@ -0,0 +1,13 @@
import { Noto_Sans, Noto_Sans_Arabic } from "next/font/google";
export const noto_sans = Noto_Sans({
subsets: ["latin", "latin-ext", "cyrillic", "cyrillic-ext", "devanagari", "vietnamese"],
display: "swap",
variable: "--font-noto-sans",
});
export const noto_sans_arabic = Noto_Sans_Arabic({
subsets: ["arabic"],
display: "swap",
variable: "--font-noto-sans-arabic",
});

5
src/interface/web/app/globals.css
View File

@@ -1,7 +1,6 @@
@tailwind base;
@tailwind components;
@tailwind utilities;
@import url("https://fonts.googleapis.com/css2?family=Noto+Sans+Arabic:wght@100..900&family=Noto+Sans:ital,wght@0,100..900;1,100..900&display=swap");
@layer base {
:root {
@@ -25,7 +24,7 @@
--input: 220 13% 91%;
--ring: 24.6 95% 53.1%;
--radius: 0.5rem;
--font-family: "Noto Sans", "Noto Sans Arabic", sans-serif !important;
--font-family: var(--font-noto-sans), var(--font-noto-sans-arabic), sans-serif !important;
/* Khoj Custom Colors */
--frosted-background-color: 20 13% 95%;
@@ -188,7 +187,7 @@
--border: 0 0% 9%;
--input: 0 0% 9%;
--ring: 20.5 90.2% 48.2%;
--font-family: "Noto Sans", "Noto Sans Arabic", sans-serif !important;
--font-family: var(--font-noto-sans), var(--font-noto-sans-arabic), sans-serif !important;
/* Imported from highlight.js */
pre code.hljs {

8
src/interface/web/app/layout.tsx
View File

@@ -1,9 +1,7 @@
import type { Metadata } from "next";
import { Noto_Sans } from "next/font/google";
import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "./globals.css";
const inter = Noto_Sans({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Khoj AI - Home",
description: "Your Second Brain.",
@@ -39,7 +37,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
<html lang="en">
<html lang="en" className={`${noto_sans.variable} ${noto_sans_arabic.variable}`}>
<meta
httpEquiv="Content-Security-Policy"
content="default-src 'self' https://assets.khoj.dev;
@@ -52,7 +50,7 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body>{children}</body>
</html>
);
}

8
src/interface/web/app/settings/layout.tsx
View File

@@ -1,10 +1,8 @@
import type { Metadata } from "next";
import { Noto_Sans } from "next/font/google";
import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../globals.css";
import { Toaster } from "@/components/ui/toaster";
const inter = Noto_Sans({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Khoj AI - Settings",
description: "Configure Khoj to get personalized, deeper assistance.",
@@ -34,7 +32,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
<html lang="en">
<html lang="en" className={`${noto_sans.variable} ${noto_sans_arabic.variable}`}>
<meta
httpEquiv="Content-Security-Policy"
content="default-src 'self' https://assets.khoj.dev;
@@ -46,7 +44,7 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>
<body>
{children}
<Toaster />
</body>

8
src/interface/web/app/share/chat/layout.tsx
View File

@@ -1,9 +1,7 @@
import type { Metadata } from "next";
import { Noto_Sans } from "next/font/google";
import { noto_sans, noto_sans_arabic } from "@/app/fonts";
import "../../globals.css";
const inter = Noto_Sans({ subsets: ["latin"] });
export const metadata: Metadata = {
title: "Khoj AI - Chat",
description: "Use this page to view a chat with Khoj AI.",
@@ -15,7 +13,7 @@ export default function RootLayout({
children: React.ReactNode;
}>) {
return (
<html lang="en">
<html lang="en" className={`${noto_sans.variable} ${noto_sans_arabic.variable}`}>
<meta
httpEquiv="Content-Security-Policy"
content="default-src 'self' https://assets.khoj.dev;
@@ -27,7 +25,7 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>
<body>
{children}
<script
dangerouslySetInnerHTML={{

8
src/interface/web/package.json
View File

@@ -1,6 +1,6 @@
{
"name": "khoj-ai",
"version": "1.30.2",
"version": "1.30.10",
"private": true,
"scripts": {
"dev": "next dev",
@@ -62,6 +62,9 @@
"react-hook-form": "^7.52.1",
"shadcn-ui": "^0.8.0",
"swr": "^2.2.5",
"tailwind-merge": "^2.3.0",
"tailwindcss": "^3.4.6",
"tailwindcss-animate": "^1.0.7",
"typescript": "^5",
"vaul": "^0.9.1",
"zod": "^3.23.8"
@@ -82,9 +85,6 @@
"lint-staged": "^15.2.7",
"nodemon": "^3.1.3",
"prettier": "3.3.3",
"tailwind-merge": "^2.3.0",
"tailwindcss": "^3.4.6",
"tailwindcss-animate": "^1.0.7",
"typescript": "^5"
},
"prettier": {

3
src/interface/web/tailwind.config.ts
View File

@@ -55,6 +55,9 @@ const config = {
},
},
extend: {
fontFamily: {
sans: ["var(--font-noto-sans)", "var(--font-noto-sans-arabic)"],
},
colors: {
border: "hsl(var(--border))",
input: "hsl(var(--input))",

11
src/khoj/interface/web/.well-known/assetlinks.json Normal file
View File

@@ -0,0 +1,11 @@
[{
"relation": ["delegate_permission/common.handle_all_urls"],
"target": {
"namespace": "android_app",
"package_name": "dev.khoj.app",
"sha256_cert_fingerprints": [
"CC:98:4A:0A:F1:CC:84:26:AC:02:86:49:AA:69:64:B9:5E:63:A3:EF:18:56:EA:CA:13:C1:3A:15:CA:49:77:46",
"D4:5A:6F:6C:18:28:D2:1C:78:27:92:C6:AC:DB:4C:12:C4:52:A1:88:9B:A1:F5:67:D1:22:FE:A0:0F:B1:AE:92"
]
}
}]

5
src/khoj/processor/conversation/anthropic/anthropic_chat.py
View File

@@ -1,9 +1,8 @@
import json
import logging
import re
from datetime import datetime, timedelta
from typing import Dict, Optional
import pyjson5
from langchain.schema import ChatMessage
from khoj.database.models import Agent, ChatModelOptions, KhojUser
@@ -110,7 +109,7 @@ def extract_questions_anthropic(
# Extract, Clean Message from Claude's Response
try:
response = clean_json(response)
response = json.loads(response)
response = pyjson5.loads(response)
response = [q.strip() for q in response["queries"] if q.strip()]
if not isinstance(response, list) or not response:
logger.error(f"Invalid response for constructing subqueries: {response}")

5
src/khoj/processor/conversation/google/gemini_chat.py
View File

@@ -1,9 +1,8 @@
import json
import logging
import re
from datetime import datetime, timedelta
from typing import Dict, Optional
import pyjson5
from langchain.schema import ChatMessage
from khoj.database.models import Agent, ChatModelOptions, KhojUser
@@ -104,7 +103,7 @@ def extract_questions_gemini(
# Extract, Clean Message from Gemini's Response
try:
response = clean_json(response)
response = json.loads(response)
response = pyjson5.loads(response)
response = [q.strip() for q in response["queries"] if q.strip()]
if not isinstance(response, list) or not response:
logger.error(f"Invalid response for constructing subqueries: {response}")

6
src/khoj/processor/conversation/offline/chat_model.py
View File

@@ -5,6 +5,7 @@ from datetime import datetime, timedelta
from threading import Thread
from typing import Any, Iterator, List, Optional, Union
import pyjson5
from langchain.schema import ChatMessage
from llama_cpp import Llama
@@ -13,6 +14,7 @@ from khoj.processor.conversation import prompts
from khoj.processor.conversation.offline.utils import download_model
from khoj.processor.conversation.utils import (
ThreadedGenerator,
clean_json,
commit_conversation_trace,
generate_chatml_messages_with_context,
messages_to_print,
@@ -114,8 +116,8 @@ def extract_questions_offline(
# Extract and clean the chat model's response
try:
response = response.strip(empty_escape_sequences)
response = json.loads(response)
response = clean_json(empty_escape_sequences)
response = pyjson5.loads(response)
questions = [q.strip() for q in response["queries"] if q.strip()]
questions = filter_questions(questions)
except:

4
src/khoj/processor/conversation/openai/gpt.py
View File

@@ -1,8 +1,8 @@
import json
import logging
from datetime import datetime, timedelta
from typing import Dict, Optional
import pyjson5
from langchain.schema import ChatMessage
from khoj.database.models import Agent, ChatModelOptions, KhojUser
@@ -104,7 +104,7 @@ def extract_questions(
# Extract, Clean Message from GPT's Response
try:
response = clean_json(response)
response = json.loads(response)
response = pyjson5.loads(response)
response = [q.strip() for q in response["queries"] if q.strip()]
if not isinstance(response, list) or not response:
logger.error(f"Invalid response for constructing subqueries: {response}")

8
src/khoj/processor/conversation/prompts.py
View File

@@ -11,11 +11,9 @@ You were created by Khoj Inc. with the following capabilities:
- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
- Users can share files and other information with you using the Khoj Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
- You *CAN* generate images, look-up real-time information from the internet, set reminders and answer questions based on the user's notes.
- Say "I don't know" or "I don't understand" if you don't know what to say or if you don't know the answer to a question.
- Make sure to use the specific LaTeX math mode delimiters for your response. LaTex math mode specific delimiters as following
- inline math mode : \\( and \\)
- display math mode: insert linebreak after opening $$, \\[ and before closing $$, \\]
- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations.
- Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay".
- Provide inline references to quotes from the user's notes or any web pages you refer to in your responses in markdown format. For example, "The farmer had ten sheep. [1](https://example.com)". *ALWAYS CITE YOUR SOURCES AND PROVIDE REFERENCES*. Add them inline to directly support your claim.
@@ -32,11 +30,9 @@ You were created by Khoj Inc. with the following capabilities:
- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
- Users can share files and other information with you using the Khoj Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
- Say "I don't know" or "I don't understand" if you don't know what to say or if you don't know the answer to a question.
- Make sure to use the specific LaTeX math mode delimiters for your response. LaTex math mode specific delimiters as following
- inline math mode : `\\(` and `\\)`
- display math mode: insert linebreak after opening `$$`, `\\[` and before closing `$$`, `\\]`
- Ask crisp follow-up questions to get additional context, when the answer cannot be inferred from the provided notes or past conversations.
- Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay".
Today is {day_of_week}, {current_date} in UTC.
@@ -50,6 +46,10 @@ Instructions:\n{bio}
gemini_verbose_language_personality = """
All questions should be answered comprehensively with details, unless the user requests a concise response specifically.
Respond in the same language as the query. Use markdown to format your responses.
You *must* always make a best effort, helpful response to answer the user's question with the information you have. You may ask necessary, limited follow-up questions to clarify the user's intent.
You must always provide a response to the user's query, even if imperfect. Do the best with the information you have, without relying on follow-up questions.
""".strip()
## General Conversation

43
src/khoj/processor/conversation/utils.py
View File

@@ -5,6 +5,7 @@ import math
import mimetypes
import os
import queue
import re
import uuid
from dataclasses import dataclass
from datetime import datetime
@@ -14,6 +15,7 @@ from time import perf_counter
from typing import Any, Callable, Dict, List, Optional
import PIL.Image
import pyjson5
import requests
import tiktoken
import yaml
@@ -538,6 +540,47 @@ def clean_code_python(code: str):
return code.strip().removeprefix("```python").removesuffix("```")
def load_complex_json(json_str):
"""
Preprocess a raw JSON string to escape unescaped double quotes within value strings,
while preserving the JSON structure and already escaped quotes.
"""
def replace_unescaped_quotes(match):
# Get the content between colons and commas/end braces
content = match.group(1)
# Replace unescaped double, single quotes that aren't already escaped
# Uses negative lookbehind to avoid replacing already escaped quotes
# Replace " with \"
processed_dq = re.sub(r'(?<!\\)"', '\\"', content)
# Replace \' with \\'
processed_final = re.sub(r"(?<!\\)\\'", r"\\\\'", processed_dq)
return f': "{processed_final}"'
# Match content between : and either , or }
# This pattern looks for ': ' followed by any characters until , or }
pattern = r':\s*"(.*?)(?<!\\)"(?=[,}])'
# Process the JSON string
cleaned = clean_json(rf"{json_str}")
processed = re.sub(pattern, replace_unescaped_quotes, cleaned)
# See which json loader can load the processed JSON as valid
errors = []
json_loaders_to_try = [json.loads, pyjson5.loads]
for loads in json_loaders_to_try:
try:
return loads(processed)
except (json.JSONDecodeError, pyjson5.Json5Exception) as e:
errors.append(f"{type(e).__name__}: {str(e)}")
# If all loaders fail, raise the aggregated error
raise ValueError(
f"Failed to load JSON with errors: {'; '.join(errors)}\n\n"
f"While attempting to load this cleaned JSON:\n{processed}"
)
def defilter_query(query: str):
"""Remove any query filters in query"""
defiltered_query = query

6
src/khoj/processor/tools/run_code.py
View File

@@ -1,6 +1,5 @@
import base64
import datetime
import json
import logging
import mimetypes
import os
@@ -15,8 +14,8 @@ from khoj.processor.conversation import prompts
from khoj.processor.conversation.utils import (
ChatEvent,
clean_code_python,
clean_json,
construct_chat_history,
load_complex_json,
)
from khoj.routers.helpers import send_message_to_model_wrapper
from khoj.utils.helpers import is_none_or_empty, timer, truncate_code_context
@@ -135,8 +134,7 @@ async def generate_python_code(
)
# Validate that the response is a non-empty, JSON-serializable list
response = clean_json(response)
response = json.loads(response)
response = load_complex_json(response)
code = response.get("code", "").strip()
input_files = response.get("input_files", [])
input_links = response.get("input_links", [])

2
src/khoj/routers/api_chat.py
View File

@@ -703,7 +703,7 @@ async def chat(
chat_metadata["agent"] = conversation.agent.slug if conversation and conversation.agent else None
chat_metadata["latency"] = f"{latency:.3f}"
chat_metadata["ttft_latency"] = f"{ttft:.3f}"
chat_metadata["usage"] = tracer.get("usage")
chat_metadata["cost"] = f"{cost:.5f}"
logger.info(f"Chat response time to first token: {ttft:.3f} seconds")
logger.info(f"Chat response total time: {latency:.3f} seconds")

3
src/khoj/routers/helpers.py
View File

@@ -27,6 +27,7 @@ from typing import (
from urllib.parse import parse_qs, quote, unquote, urljoin, urlparse
import cron_descriptor
import pyjson5
import pytz
import requests
from apscheduler.job import Job
@@ -541,7 +542,7 @@ async def generate_online_subqueries(
# Validate that the response is a non-empty, JSON-serializable list
try:
response = clean_json(response)
response = json.loads(response)
response = pyjson5.loads(response)
response = {q.strip() for q in response["queries"] if q.strip()}
if not isinstance(response, set) or not response or len(response) == 0:
logger.error(f"Invalid response for constructing subqueries: {response}. Returning original query: {q}")

6
src/khoj/routers/research.py
View File

@@ -1,4 +1,3 @@
import json
import logging
from datetime import datetime
from typing import Callable, Dict, List, Optional
@@ -10,10 +9,10 @@ from khoj.database.models import Agent, KhojUser
from khoj.processor.conversation import prompts
from khoj.processor.conversation.utils import (
InformationCollectionIteration,
clean_json,
construct_chat_history,
construct_iteration_history,
construct_tool_chat_history,
load_complex_json,
)
from khoj.processor.tools.online_search import read_webpages, search_online
from khoj.processor.tools.run_code import run_code
@@ -106,8 +105,7 @@ async def apick_next_tool(
return
try:
response = clean_json(response)
response = json.loads(response)
response = load_complex_json(response)
selected_tool = response.get("tool", None)
generated_query = response.get("query", None)
scratchpad = response.get("scratchpad", None)

5
src/khoj/routers/web_client.py
View File

@@ -134,3 +134,8 @@ def automations_config_page(
"request": request,
},
)
@web_client.get("/.well-known/assetlinks.json", response_class=FileResponse)
def assetlinks(request: Request):
return FileResponse(constants.assetlinks_file_path)

1
src/khoj/utils/constants.py
View File

@@ -5,6 +5,7 @@ app_root_directory = Path(__file__).parent.parent.parent
web_directory = app_root_directory / "khoj/interface/web/"
next_js_directory = app_root_directory / "khoj/interface/built/"
pypi_static_directory = app_root_directory / "khoj/interface/compiled/"
assetlinks_file_path = web_directory / ".well-known/assetlinks.json"
empty_escape_sequences = "\n|\r|\t| "
app_env_filepath = "~/.khoj/env"
telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"

6
src/khoj/utils/helpers.py
View File

@@ -256,9 +256,9 @@ def get_server_id():
def telemetry_disabled(app_config: AppConfig, telemetry_disable_env) -> bool:
return (
not app_config.should_log_telemetry if app_config and app_config.should_log_telemetry else telemetry_disable_env
)
if telemetry_disable_env is True:
return True
return not app_config or not app_config.should_log_telemetry
def log_telemetry(

154
tests/evals/eval.py
View File

@@ -3,8 +3,10 @@ import concurrent.futures
import json
import logging
import os
import re
import time
from datetime import datetime
from functools import partial
from io import StringIO
from textwrap import dedent
from threading import Lock
@@ -24,13 +26,10 @@ logger = logging.getLogger(__name__)
KHOJ_URL = os.getenv("KHOJ_URL", "http://localhost:42110")
KHOJ_CHAT_API_URL = f"{KHOJ_URL}/api/chat"
KHOJ_API_KEY = os.getenv("KHOJ_API_KEY")
KHOJ_MODE = os.getenv("KHOJ_MODE", "default") # E.g research, general, notes etc.
KHOJ_MODE = os.getenv("KHOJ_MODE", "default").lower() # E.g research, general, notes etc.
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
GEMINI_EVAL_MODEL = os.getenv("GEMINI_EVAL_MODEL", "gemini-1.5-pro-002")
GEMINI_API_URL = (
f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_EVAL_MODEL}:generateContent?key={GEMINI_API_KEY}"
)
SAMPLE_SIZE = os.getenv("SAMPLE_SIZE") # Number of examples to evaluate
RANDOMIZE = os.getenv("RANDOMIZE", "false").lower() == "true" # Randomize examples
@@ -128,6 +127,99 @@ def load_simpleqa_dataset():
return None
def load_gpqa_dataset():
"""
Load the Google GPQA benchmark dataset from HuggingFace
GPQA is a benchmark dataset to evaluate retrieval and answering capabilities of agents.
It contains ~800 requiring multi-hop retrieval and reasoning across various topics.
### Data Fields
- Prompt: The question to be answered
- Answer: The ground truth answer
- reasoning_types: The type of reasoning required to answer the question
"""
import random
def format_multiple_choice_question(row: Dict) -> tuple[str, str]:
"""
Create GPQA multi-choice prompt from shuffled answer choices and question.
Refer: https://github.com/openai/simple-evals/blob/a8e85cc8a5dea497d915f870895250e07f9cc737/common.py#L12
Returns formatted prompt and correct answer letter.
"""
# Gather choices
choices = [
row["Incorrect Answer 1"],
row["Incorrect Answer 2"],
row["Incorrect Answer 3"],
row["Correct Answer"],
]
# Shuffle choices
random.shuffle(choices)
# Get correct answer letter
correct_index = choices.index(row["Correct Answer"])
correct_letter = "ABCD"[correct_index]
prompt = f"""
Answer the following multiple choice question. Answer should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.
{row["Question"]}
A) {choices[0]}
B) {choices[1]}
C) {choices[2]}
D) {choices[3]}
""".strip()
return prompt, correct_letter
try:
dataset = load_dataset("Idavidrein/gpqa", "gpqa_diamond", split="train")
# Create multi-choice q&a prompt from choices and correct answer
prompts_and_answers = [format_multiple_choice_question(row) for row in dataset]
# Normalize dataset to FRAMES format
dataset = dataset.rename_columns({"Subdomain": "reasoning_types"})
dataset = dataset.add_column("Prompt", [p[0] for p in prompts_and_answers])
dataset = dataset.add_column("Answer", [p[1] for p in prompts_and_answers])
# Sample and shuffle dataset if configured
dataset = dataset.shuffle() if RANDOMIZE else dataset
dataset = dataset[: int(SAMPLE_SIZE)] if SAMPLE_SIZE else dataset
return dataset
except Exception as e:
logger.error(f"Error loading dataset: {e}")
return None
def load_math500_dataset():
"""
Load and format the MATH500 dataset to match the evaluation script's structure.
Args:
sample_size (int, optional): Number of samples to include. Defaults to None (use full dataset).
randomize (bool, optional): Whether to randomize the dataset. Defaults to False.
Returns:
Dataset: Formatted HuggingFace Dataset.
"""
try:
# Load the MATH500 dataset from HuggingFace
dataset = load_dataset("HuggingFaceH4/MATH-500", split="test")
dataset = dataset.rename_columns({"problem": "Prompt", "answer": "Answer", "subject": "reasoning_types"})
dataset = dataset.shuffle() if RANDOMIZE else dataset
dataset = dataset.select(range(int(SAMPLE_SIZE))) if SAMPLE_SIZE else dataset
return dataset
except Exception as e:
print(f"Error loading and formatting MATH500 dataset: {e}")
return None
def get_agent_response(prompt: str) -> Dict[str, Any]:
"""Get response from the Khoj API"""
# Set headers
@@ -152,7 +244,30 @@ def get_agent_response(prompt: str) -> Dict[str, Any]:
return {"response": "", "usage": {}}
def evaluate_response(query: str, agent_response: str, ground_truth: str) -> tuple[bool | None, str, float]:
def evaluate_response_with_mcq_match(
query: str, agent_response: str, ground_truth: str
) -> tuple[bool | None, str, float]:
"""Evaluate Khoj response against benchmark ground truth using string matching"""
try:
# Extract answer from agent response
answer_pattern_multichoice = r"(?i)Answer\s*:\s*([A-D])"
match = re.search(answer_pattern_multichoice, agent_response)
extracted_answer = match.group(1) if match else None
# Check if extracted answer matches ground truth
decision = extracted_answer == ground_truth
explanation = f"Agent response {'matches' if decision else 'does not match'} ground truth {ground_truth}"
# Return decision, explanation and cost in structured form
return decision, explanation, 0.0
except Exception as e:
logger.error(f"Error in evaluation: {e}")
return None, f"Evaluation failed: {str(e)}", 0.0
def evaluate_response_with_gemini(
query: str, agent_response: str, ground_truth: str, eval_model=GEMINI_EVAL_MODEL
) -> tuple[bool | None, str, float]:
"""Evaluate Khoj response against benchmark ground truth using Gemini"""
evaluation_prompt = f"""
Compare the following agent response with the ground truth answer.
@@ -166,10 +281,13 @@ def evaluate_response(query: str, agent_response: str, ground_truth: str) -> tup
Provide your evaluation in the following json format:
{"explanation:" "[How you made the decision?)", "decision:" "(TRUE if response contains key information, FALSE otherwise)"}
"""
gemini_api_url = (
f"https://generativelanguage.googleapis.com/v1beta/models/{eval_model}:generateContent?key={GEMINI_API_KEY}"
)
try:
response = requests.post(
GEMINI_API_URL,
gemini_api_url,
headers={"Content-Type": "application/json"},
json={
"contents": [{"parts": [{"text": evaluation_prompt}]}],
@@ -182,7 +300,7 @@ def evaluate_response(query: str, agent_response: str, ground_truth: str) -> tup
# Update cost of evaluation
input_tokens = response_json["usageMetadata"]["promptTokenCount"]
ouput_tokens = response_json["usageMetadata"]["candidatesTokenCount"]
cost = get_cost_of_chat_message(GEMINI_EVAL_MODEL, input_tokens, ouput_tokens)
cost = get_cost_of_chat_message(eval_model, input_tokens, ouput_tokens)
# Parse evaluation response
eval_response: dict[str, str] = json.loads(
@@ -200,7 +318,7 @@ def evaluate_response(query: str, agent_response: str, ground_truth: str) -> tup
return None, f"Evaluation failed: {str(e)}", 0.0
def process_batch(batch, batch_start, results, dataset_length):
def process_batch(batch, batch_start, results, dataset_length, response_evaluator):
global running_cost
for idx, (prompt, answer, reasoning_type) in enumerate(batch):
current_index = batch_start + idx
@@ -219,7 +337,7 @@ def process_batch(batch, batch_start, results, dataset_length):
decision = None
explanation = "Agent response is empty. This maybe due to a service error."
else:
decision, explanation, eval_cost = evaluate_response(prompt, agent_response, answer)
decision, explanation, eval_cost = response_evaluator(prompt, agent_response, answer)
# Store results
results.append(
@@ -292,7 +410,7 @@ def parse_args():
"--dataset",
"-d",
default="frames",
choices=["frames", "simpleqa"],
choices=["frames", "simpleqa", "gpqa", "math500"],
help="Dataset to use for evaluation (default: frames)",
)
return parser.parse_args()
@@ -309,12 +427,24 @@ def main():
dataset = load_frames_dataset()
elif args.dataset == "simpleqa":
dataset = load_simpleqa_dataset()
elif args.dataset == "gpqa":
dataset = load_gpqa_dataset()
elif args.dataset == "math500":
dataset = load_math500_dataset()
if dataset is None:
return
# Initialize variables
results = []
dataset_length = len(dataset["Prompt"])
if args.dataset == "gpqa":
response_evaluator = evaluate_response_with_mcq_match
elif args.dataset == "math500":
response_evaluator = partial(
evaluate_response_with_gemini, eval_model=os.getenv("GEMINI_EVAL_MODEL", "gemini-1.5-flash-002")
)
else:
response_evaluator = evaluate_response_with_gemini
# Process examples in batches
with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -326,7 +456,9 @@ def main():
dataset["Answer"][i : i + BATCH_SIZE],
dataset["reasoning_types"][i : i + BATCH_SIZE],
)
futures.append(executor.submit(process_batch, batch, batch_start, results, dataset_length))
futures.append(
executor.submit(process_batch, batch, batch_start, results, dataset_length, response_evaluator)
)
# Wait for all futures to complete
concurrent.futures.wait(futures)

12
tests/test_conversation_utils.py
View File

@@ -104,6 +104,18 @@ class TestTruncateMessage:
assert truncated_chat_history[0] != copy_big_chat_message
def test_load_complex_raw_json_string():
# Arrange
raw_json = r"""{"key": "value with unescaped " and unescaped \' and escaped \" and escaped \\'"}"""
expeced_json = {"key": "value with unescaped \" and unescaped \\' and escaped \" and escaped \\'"}
# Act
parsed_json = utils.load_complex_json(raw_json)
# Assert
assert parsed_json == expeced_json
def generate_content(count):
return " ".join([f"{index}" for index, _ in enumerate(range(count))])

10
versions.json
View File

@@ -93,5 +93,13 @@
"1.29.1": "0.15.0",
"1.30.0": "0.15.0",
"1.30.1": "0.15.0",
"1.30.2": "0.15.0"
"1.30.2": "0.15.0",
"1.30.3": "0.15.0",
"1.30.4": "0.15.0",
"1.30.5": "0.15.0",
"1.30.6": "0.15.0",
"1.30.7": "0.15.0",
"1.30.8": "0.15.0",
"1.30.9": "0.15.0",
"1.30.10": "0.15.0"
}