feat(data-exploration): data exploration backend API with types

frontend/src/lib/api.ts

@@ -49,6 +49,7 @@ import { ActivityLogItem, ActivityScope } from 'lib/components/ActivityLog/human
 import { ActivityLogProps } from 'lib/components/ActivityLog/ActivityLog'
 import { SavedSessionRecordingPlaylistsResult } from 'scenes/session-recordings/saved-playlists/savedSessionRecordingPlaylistsLogic'
 import { dayjs } from 'lib/dayjs'
+import { QuerySchema } from '~/queries/schema'
 
 export const ACTIVITY_PAGE_SIZE = 20
 
@@ -424,6 +425,11 @@ class ApiRequest {
             .withQueryString(toParams({ date_from: dateFrom, date_to: dateTo }))
     }
 
+    // # Queries
+    public query(teamId?: TeamType['id']): ApiRequest {
+        return this.projectsDetail(teamId).addPathComponent('query')
+    }
+
     // Request finalization
 
     public async get(options?: ApiMethodOptions): Promise<any> {
@@ -1124,6 +1130,19 @@ const api = {
         },
     },
 
+    async query<T extends Record<string, any> = QuerySchema>(
+        query: T,
+        options?: ApiMethodOptions
+    ): Promise<
+        T extends { [response: string]: any }
+            ? T['response'] extends infer P | undefined
+                ? P
+                : T['response']
+            : Record<string, any>
+    > {
+        return await new ApiRequest().query().create({ ...options, data: query })
+    },
+
     /** Fetch data from specified URL. The result already is JSON-parsed. */
     async get(url: string, options?: ApiMethodOptions): Promise<any> {
         const res = await api.getResponse(url, options)

frontend/src/queries/nodes/DataTable/ColumnConfigurator/ColumnConfigurator.tsx

@@ -16,7 +16,7 @@ import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
 import { FEATURE_FLAGS } from 'lib/constants'
 import { useState } from 'react'
 import { columnConfiguratorLogic, ColumnConfiguratorLogicProps } from './columnConfiguratorLogic'
-import { defaultDataTableColumns, extractExpressionComment } from '../utils'
+import { defaultDataTableColumns, extractExpressionComment, removeExpressionComment } from '../utils'
 import { DataTableNode, NodeKind } from '~/queries/schema'
 import { LemonModal } from 'lib/components/LemonModal'
 import { isEventsQuery, taxonomicFilterToHogQl } from '~/queries/utils'
@@ -42,10 +42,16 @@ export function ColumnConfigurator({ query, setQuery }: ColumnConfiguratorProps)
         columns: columnsInQuery,
         setColumns: (columns: string[]) => {
             if (isEventsQuery(query.source)) {
+                const oldOrderBy = query.source.orderBy
+                const firstOldOrderBy = oldOrderBy?.length == 1 ? removeExpressionComment(oldOrderBy?.[0]) : undefined
                 setQuery?.({
                     ...query,
                     source: {
                         ...query.source,
+                        orderBy:
+                            firstOldOrderBy && columns.find((c) => removeExpressionComment(c) === firstOldOrderBy)
+                                ? oldOrderBy
+                                : undefined,
                         select: columns,
                     },
                 })

frontend/src/queries/nodes/DataTable/dataTableLogic.ts

@@ -1,6 +1,6 @@
 import { actions, connect, kea, key, path, props, propsChanged, reducers, selectors } from 'kea'
 import type { dataTableLogicType } from './dataTableLogicType'
-import { AnyDataNode, DataTableNode, EventsQuery, HogQLExpression, NodeKind } from '~/queries/schema'
+import { AnyDataNode, DataTableNode, EventsQuery, NodeKind } from '~/queries/schema'
 import { getColumnsForQuery, removeExpressionComment } from './utils'
 import { objectsEqual, sortedKeys } from 'lib/utils'
 import { isDataTableNode, isEventsQuery } from '~/queries/utils'
@@ -37,7 +37,7 @@ export const dataTableLogic = kea<dataTableLogicType>([
         return props.key
     }),
     path(['queries', 'nodes', 'DataTable', 'dataTableLogic']),
-    actions({ setColumnsInQuery: (columns: HogQLExpression[]) => ({ columns }) }),
+    actions({ setColumnsInQuery: (columns: string[]) => ({ columns }) }),
     reducers(({ props }) => ({
         columnsInQuery: [getColumnsForQuery(props.query), { setColumnsInQuery: (_, { columns }) => columns }],
     })),

frontend/src/queries/nodes/DataTable/utils.ts

@@ -1,7 +1,7 @@
-import { DataNode, HogQLExpression, DataTableNode, NodeKind } from '~/queries/schema'
+import { DataNode, DataTableNode, NodeKind } from '~/queries/schema'
 import { isEventsQuery } from '~/queries/utils'
 
-export const defaultDataTableEventColumns: HogQLExpression[] = [
+export const defaultDataTableEventColumns: string[] = [
     '*',
     'event',
     'person',
@@ -10,9 +10,9 @@ export const defaultDataTableEventColumns: HogQLExpression[] = [
     'timestamp',
 ]
 
-export const defaultDataTablePersonColumns: HogQLExpression[] = ['person', 'id', 'created_at', 'person.$delete']
+export const defaultDataTablePersonColumns: string[] = ['person', 'id', 'created_at', 'person.$delete']
 
-export function defaultDataTableColumns(kind: NodeKind): HogQLExpression[] {
+export function defaultDataTableColumns(kind: NodeKind): string[] {
     return kind === NodeKind.PersonsNode
         ? defaultDataTablePersonColumns
         : kind === NodeKind.EventsQuery
@@ -22,14 +22,14 @@ export function defaultDataTableColumns(kind: NodeKind): HogQLExpression[] {
         : []
 }
 
-export function getDataNodeDefaultColumns(source: DataNode): HogQLExpression[] {
+export function getDataNodeDefaultColumns(source: DataNode): string[] {
     return (
         (isEventsQuery(source) && Array.isArray(source.select) && source.select.length > 0 ? source.select : null) ??
         defaultDataTableColumns(source.kind)
     )
 }
 
-export function getColumnsForQuery(query: DataTableNode): HogQLExpression[] {
+export function getColumnsForQuery(query: DataTableNode): string[] {
     return query.columns ?? getDataNodeDefaultColumns(query.source)
 }
 

frontend/src/queries/query.ts

@@ -37,18 +37,15 @@ export async function query<N extends DataNode = DataNode>(
 ): Promise<N['response']> {
     if (isEventsQuery(query)) {
         if (!query.before && !query.after) {
-            const earlyResults = await api.get(
-                getEventsEndpoint({ ...query, after: now().subtract(EVENTS_DAYS_FIRST_FETCH, 'day').toISOString() }),
+            const earlyResults = await api.query(
+                { ...query, after: now().subtract(EVENTS_DAYS_FIRST_FETCH, 'day').toISOString() },
                 methodOptions
             )
             if (earlyResults.results.length > 0) {
                 return earlyResults
             }
         }
-        return await api.get(
-            getEventsEndpoint({ after: now().subtract(1, 'year').toISOString(), ...query }),
-            methodOptions
-        )
+        return await api.query({ after: now().subtract(1, 'year').toISOString(), ...query }, methodOptions)
     } else if (isPersonsNode(query)) {
         return await api.get(getPersonsEndpoint(query), methodOptions)
     } else if (isInsightQueryNode(query)) {

frontend/src/queries/schema.json

@@ -1339,7 +1339,7 @@
                 "columns": {
                     "description": "Columns shown in the table, unless the `source` provides them.",
                     "items": {
-                        "$ref": "#/definitions/HogQLExpression"
+                        "type": "string"
                     },
                     "type": "array"
                 },
@@ -1354,7 +1354,7 @@
                 "hiddenColumns": {
                     "description": "Columns that aren't shown in the table, even if in columns or returned data",
                     "items": {
-                        "$ref": "#/definitions/HogQLExpression"
+                        "type": "string"
                     },
                     "type": "array"
                 },
@@ -1630,7 +1630,7 @@
             "properties": {
                 "actionId": {
                     "description": "Show events matching a given action",
-                    "type": "number"
+                    "type": "integer"
                 },
                 "after": {
                     "description": "Only fetch events that happened after this timestamp",
@@ -1657,11 +1657,11 @@
                 },
                 "limit": {
                     "description": "Number of rows to return",
-                    "type": "number"
+                    "type": "integer"
                 },
                 "offset": {
                     "description": "Number of rows to skip before returning rows",
-                    "type": "number"
+                    "type": "integer"
                 },
                 "orderBy": {
                     "description": "Columns to order by",
@@ -1714,14 +1714,14 @@
                 "select": {
                     "description": "Return a limited set of data. Required.",
                     "items": {
-                        "$ref": "#/definitions/HogQLExpression"
+                        "type": "string"
                     },
                     "type": "array"
                 },
                 "where": {
                     "description": "HogQL filters to apply on returned data",
                     "items": {
-                        "$ref": "#/definitions/HogQLExpression"
+                        "type": "string"
                     },
                     "type": "array"
                 }
@@ -1930,9 +1930,6 @@
             "const": "unique_group",
             "type": "string"
         },
-        "HogQLExpression": {
-            "type": "string"
-        },
         "InsightQueryNode": {
             "anyOf": [
                 {

frontend/src/queries/schema.ts

@@ -21,6 +21,18 @@ import {
     LifecycleToggle,
 } from '~/types'
 
+/**
+ * PostHog Query Schema definition.
+ *
+ * This file acts as the source of truth for:
+ *
+ * - frontend/src/queries/schema.json
+ *   - generated from typescript via "pnpm run generate:schema:json"
+ *
+ * - posthog/schema.py
+ *   - generated from json the above json via "pnpm run generate:schema:python"
+ * */
+
 export enum NodeKind {
     // Data nodes
     EventsNode = 'EventsNode',
@@ -128,20 +140,29 @@ export interface NewEntityNode extends EntityNode {
 export interface EventsQuery extends DataNode {
     kind: NodeKind.EventsQuery
     /** Return a limited set of data. Required. */
-    select: HogQLExpression[]
+    select: string[]
     /** HogQL filters to apply on returned data */
-    where?: HogQLExpression[]
+    where?: string[]
     /** Properties configurable in the interface */
     properties?: AnyPropertyFilter[]
     /** Fixed properties in the query, can't be edited in the interface (e.g. scoping down by person) */
     fixedProperties?: AnyPropertyFilter[]
     /** Limit to events matching this string */
     event?: string
-    /** Number of rows to return */
+    /**
+     * Number of rows to return
+     * @asType integer
+     */
     limit?: number
-    /** Number of rows to skip before returning rows */
+    /**
+     * Number of rows to skip before returning rows
+     * @asType integer
+     */
     offset?: number
-    /** Show events matching a given action */
+    /**
+     * Show events matching a given action
+     * @asType integer
+     */
     actionId?: number
     /** Show events for a given person */
     personId?: string
@@ -181,9 +202,9 @@ export interface DataTableNode extends Node {
     /** Source of the events */
     source: EventsNode | EventsQuery | PersonsNode | RecentPerformancePageViewNode
     /** Columns shown in the table, unless the `source` provides them. */
-    columns?: HogQLExpression[]
+    columns?: string[]
     /** Columns that aren't shown in the table, even if in columns or returned data */
-    hiddenColumns?: HogQLExpression[]
+    hiddenColumns?: string[]
     /** Show with most visual options enabled. Used in scenes. */
     full?: boolean
     /** Include an event filter above the table (EventsNode only) */
@@ -357,8 +378,6 @@ export interface RecentPerformancePageViewNode extends DataNode {
     numberOfDays?: number // defaults to 7
 }
 
-export type HogQLExpression = string
-
 // Legacy queries
 
 export interface LegacyQuery extends Node {

package.json

@@ -27,7 +27,9 @@
         "clean": "rm -rf frontend/dist && mkdir frontend/dist",
         "build": "pnpm copy-scripts && pnpm build:esbuild",
         "build:esbuild": "node frontend/build.mjs",
-        "schema:build": "ts-json-schema-generator -f tsconfig.json --path 'frontend/src/*.ts' --type 'QuerySchema' --no-type-check > frontend/src/queries/schema.json && prettier --write frontend/src/queries/schema.json",
+        "schema:build": "pnpm run schema:build:json && pnpm run schema:build:python",
+        "schema:build:json": "ts-json-schema-generator -f tsconfig.json --path 'frontend/src/*.ts' --type 'QuerySchema' --no-type-check > frontend/src/queries/schema.json && prettier --write frontend/src/queries/schema.json",
+        "schema:build:python": "datamodel-codegen --input frontend/src/queries/schema.json --input-file-type jsonschema --output posthog/schema.py && black posthog/schema.py",
         "packages:build": "pnpm packages:build:apps-common && pnpm packages:build:lemon-ui",
         "packages:build:apps-common": "cd frontend/@posthog/apps-common && pnpm i && pnpm build",
         "packages:build:lemon-ui": "cd frontend/@posthog/lemon-ui && pnpm i && pnpm build",

posthog/api/__init__.py

@@ -27,6 +27,7 @@
     plugin_log_entry,
     prompt,
     property_definition,
+    query,
     sharing,
     site_app,
     tagged_item,
@@ -65,6 +66,7 @@ def api_not_found(request):
     "project_plugins_config_logs",
     ["team_id", "plugin_config_id"],
 )
+
 projects_router.register(r"annotations", annotation.AnnotationsViewSet, "project_annotations", ["team_id"])
 projects_router.register(
     r"activity_log",
@@ -148,6 +150,7 @@ def api_not_found(request):
 projects_router.register(r"uploaded_media", uploaded_media.MediaViewSet, "project_media", ["team_id"])
 
 projects_router.register(r"tags", tagged_item.TaggedItemViewSet, "project_tags", ["team_id"])
+projects_router.register(r"query", query.QueryViewSet, "project_query", ["team_id"])
 
 # General endpoints (shared across CH & PG)
 router.register(r"login", authentication.LoginViewSet)