Skip to content

Commit

Permalink
npx convex self-host data (#34111)
Browse files Browse the repository at this point in the history 
as with `npx convex self-host env`, import, run, etc. add a subcommand for querying data via the CLI for self-hosted deployments.

GitOrigin-RevId: 02a6c9c97dcab06f8b54751b9efa1c8f0278dba9
  • Loading branch information
@ldanilek
ldanilek authored and Convex, Inc. committed Feb 6, 2025
1 parent e41f994 commit f284b95
Show file tree
Hide file tree
Showing 4 changed files with 304 additions and 230 deletions.
242 changes: 14 additions & 228 deletions npm-packages/convex/src/cli/data.ts
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,12 @@
import { Option } from "@commander-js/extra-typings";
import chalk from "chalk";
import {
Context,
logError,
logOutput,
logWarning,
oneoffContext,
} from "../bundler/context.js";
import { Base64 } from "../values/index.js";
import { Value } from "../values/value.js";
import { oneoffContext } from "../bundler/context.js";
import {
deploymentSelectionFromOptions,
fetchDeploymentCredentialsProvisionProd,
} from "./lib/api.js";
import { runSystemPaginatedQuery } from "./lib/run.js";
import { parsePositiveInteger } from "./lib/utils/utils.js";
import { Command } from "@commander-js/extra-typings";
import { actionDescription } from "./lib/command.js";
import { dataInDeployment } from "./lib/data.js";

export const data = new Command("data")
.summary("List tables and print data from your database")
Expand All @@ -27,30 +17,7 @@ export const data = new Command("data")
"By default, this inspects your dev deployment.",
)
.allowExcessArguments(false)
.argument("[table]", "If specified, list documents in this table.")
.addOption(
new Option(
"--limit <n>",
"List only the `n` the most recently created documents.",
)
.default(100)
.argParser(parsePositiveInteger),
)
.addOption(
new Option(
"--order <choice>",
"Order the documents by their `_creationTime`.",
)
.choices(["asc", "desc"])
.default("desc"),
)
.addOption(
new Option(
"--component <path>",
"Path to the component in the component tree defined in convex.config.ts.\n" +
" By default, inspects data in the root component",
).hideHelp(),
)
.addDataOptions()
.addDeploymentSelectionOptions(actionDescription("Inspect the database in"))
.showHelpAfterError()
.action(async (tableName, options) => {
Expand All @@ -63,196 +30,15 @@ export const data = new Command("data")
deploymentName,
} = await fetchDeploymentCredentialsProvisionProd(ctx, deploymentSelection);

if (tableName !== undefined) {
await listDocuments(ctx, deploymentUrl, adminKey, tableName, {
...options,
order: options.order as "asc" | "desc",
componentPath: options.component ?? "",
});
} else {
await listTables(
ctx,
deploymentUrl,
adminKey,
deploymentName,
options.component ?? "",
);
}
});

async function listTables(
ctx: Context,
deploymentUrl: string,
adminKey: string,
deploymentName: string | undefined,
componentPath: string,
) {
const tables = (await runSystemPaginatedQuery(ctx, {
deploymentUrl,
adminKey,
functionName: "_system/cli/tables",
componentPath,
args: {},
})) as { name: string }[];
if (tables.length === 0) {
logError(
ctx,
`There are no tables in the ${
deploymentName ? `${chalk.bold(deploymentName)} deployment's ` : ""
}database.`,
);
return;
}
const tableNames = tables.map((table) => table.name);
tableNames.sort();
logOutput(ctx, tableNames.join("\n"));
}

async function listDocuments(
ctx: Context,
deploymentUrl: string,
adminKey: string,
tableName: string,
options: {
limit: number;
order: "asc" | "desc";
componentPath: string;
},
) {
const data = (await runSystemPaginatedQuery(ctx, {
deploymentUrl,
adminKey,
functionName: "_system/cli/tableData",
componentPath: options.componentPath,
args: {
table: tableName,
order: options.order ?? "desc",
},
limit: options.limit + 1,
})) as Record<string, Value>[];

if (data.length === 0) {
logError(ctx, "There are no documents in this table.");
return;
}
const deploymentNotice = deploymentName
? `${chalk.bold(deploymentName)} deployment's `
: "";

logDocumentsTable(
ctx,
data.slice(0, options.limit).map((document) => {
const printed: Record<string, string> = {};
for (const key in document) {
printed[key] = stringify(document[key]);
}
return printed;
}),
);
if (data.length > options.limit) {
logWarning(
ctx,
chalk.yellow(
`Showing the ${options.limit} ${
options.order === "desc" ? "most recently" : "oldest"
} created document${
options.limit > 1 ? "s" : ""
}. Use the --limit option to see more.`,
),
);
}
}

function logDocumentsTable(ctx: Context, rows: Record<string, string>[]) {
const columnsToWidths: Record<string, number> = {};
for (const row of rows) {
for (const column in row) {
const value = row[column];
columnsToWidths[column] = Math.max(
value.length,
columnsToWidths[column] ?? 0,
);
}
}
const unsortedFields = Object.keys(columnsToWidths);
unsortedFields.sort();
const fields = Array.from(
new Set(["_id", "_creationTime", ...unsortedFields]),
);
const columnWidths = fields.map((field) => columnsToWidths[field]);
const lineLimit = process.stdout.isTTY ? process.stdout.columns : undefined;

let didTruncate = false;

function limitLine(line: string, limit: number | undefined) {
if (limit === undefined) {
return line;
}
const limitWithBufferForUnicode = limit - 10;
if (line.length > limitWithBufferForUnicode) {
didTruncate = true;
}
return line.slice(0, limitWithBufferForUnicode);
}

logOutput(
ctx,
limitLine(
fields.map((field, i) => field.padEnd(columnWidths[i])).join(" | "),
lineLimit,
),
);
logOutput(
ctx,
limitLine(
columnWidths.map((width) => "-".repeat(width)).join("-|-"),
lineLimit,
),
);
for (const row of rows) {
logOutput(
ctx,
limitLine(
fields
.map((field, i) => (row[field] ?? "").padEnd(columnWidths[i]))
.join(" | "),
lineLimit,
),
);
}
if (didTruncate) {
logWarning(
ctx,
chalk.yellow(
"Lines were truncated to fit the terminal width. Pipe the command to see " +
"the full output, such as:\n `npx convex data tableName | less -S`",
),
);
}
}

function stringify(value: Value): string {
if (value === null) {
return "null";
}
if (typeof value === "bigint") {
return `${value.toString()}n`;
}
if (typeof value === "number") {
return value.toString();
}
if (typeof value === "boolean") {
return value.toString();
}
if (typeof value === "string") {
return JSON.stringify(value);
}
if (value instanceof ArrayBuffer) {
const base64Encoded = Base64.fromByteArray(new Uint8Array(value));
return `Bytes("${base64Encoded}")`;
}
if (value instanceof Array) {
return `[${value.map(stringify).join(", ")}]`;
}
const pairs = Object.entries(value)
.map(([k, v]) => `"${k}": ${stringify(v!)}`)
.join(", ");
return `{ ${pairs} }`;
}
await dataInDeployment(ctx, {
deploymentUrl,
adminKey,
deploymentNotice,
tableName,
...options,
});
});
40 changes: 40 additions & 0 deletions npm-packages/convex/src/cli/lib/command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { OneoffCtx } from "../../bundler/context.js";
import {
CONVEX_SELF_HOST_ADMIN_KEY_VAR_NAME,
CONVEX_SELF_HOST_URL_VAR_NAME,
parsePositiveInteger,
} from "./utils/utils.js";

declare module "@commander-js/extra-typings" {
Expand Down Expand Up @@ -132,6 +133,18 @@ declare module "@commander-js/extra-typings" {
includeFileStorage?: boolean;
}
>;

/**
* Adds options for the `data` command.
*/
addDataOptions(): Command<
[...Args, string | undefined],
Opts & {
limit: number;
order: "asc" | "desc";
component?: string;
}
>;
}
}

Expand Down Expand Up @@ -451,3 +464,30 @@ Command.prototype.addExportOptions = function () {
),
);
};

Command.prototype.addDataOptions = function () {
return this.addOption(
new Option(
"--limit <n>",
"List only the `n` the most recently created documents.",
)
.default(100)
.argParser(parsePositiveInteger),
)
.addOption(
new Option(
"--order <choice>",
"Order the documents by their `_creationTime`.",
)
.choices(["asc", "desc"])
.default("desc"),
)
.addOption(
new Option(
"--component <path>",
"Path to the component in the component tree defined in convex.config.ts.\n" +
" By default, inspects data in the root component",
).hideHelp(),
)
.argument("[table]", "If specified, list documents in this table.");
};
Loading

0 comments on commit f284b95

Please sign in to comment.