Skip to content

Commit

Permalink
Better support for JSON files describing commands metadata (#202)
Browse files Browse the repository at this point in the history
* Improved JSON 'arguments' fallback in gencommand.py (now
  independent of the 'arity' field).
* Update generated command table from latest Redis.
* Support JSON generated by redis utils/generate-commands-json.py
  and the commands.json in the redis-doc repo.
* Update comments in gencommands.py about the JSON file format.
  • Loading branch information
zuiderkwast authored Feb 19, 2024
1 parent 9858f64 commit 88f9487
Show file tree
Hide file tree
Showing 2 changed files with 60 additions and 28 deletions.
5 changes: 4 additions & 1 deletion cmddef.h
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,10 @@ COMMAND(CLIENT_INFO, "CLIENT", "INFO", 2, NONE, 0)
COMMAND(CLIENT_KILL, "CLIENT", "KILL", -3, NONE, 0)
COMMAND(CLIENT_LIST, "CLIENT", "LIST", -2, NONE, 0)
COMMAND(CLIENT_NO_EVICT, "CLIENT", "NO-EVICT", 3, NONE, 0)
COMMAND(CLIENT_NO_TOUCH, "CLIENT", "NO-TOUCH", 3, NONE, 0)
COMMAND(CLIENT_PAUSE, "CLIENT", "PAUSE", -3, NONE, 0)
COMMAND(CLIENT_REPLY, "CLIENT", "REPLY", 3, NONE, 0)
COMMAND(CLIENT_SETINFO, "CLIENT", "SETINFO", 4, NONE, 0)
COMMAND(CLIENT_SETNAME, "CLIENT", "SETNAME", 3, NONE, 0)
COMMAND(CLIENT_TRACKING, "CLIENT", "TRACKING", -3, NONE, 0)
COMMAND(CLIENT_TRACKINGINFO, "CLIENT", "TRACKINGINFO", 2, NONE, 0)
Expand Down Expand Up @@ -256,7 +258,7 @@ COMMAND(SDIFF, "SDIFF", NULL, -2, INDEX, 1)
COMMAND(SDIFFSTORE, "SDIFFSTORE", NULL, -3, INDEX, 1)
COMMAND(SELECT, "SELECT", NULL, 2, NONE, 0)
COMMAND(SENTINEL_CKQUORUM, "SENTINEL", "CKQUORUM", 3, NONE, 0)
COMMAND(SENTINEL_CONFIG, "SENTINEL", "CONFIG", -3, NONE, 0)
COMMAND(SENTINEL_CONFIG, "SENTINEL", "CONFIG", -4, NONE, 0)
COMMAND(SENTINEL_DEBUG, "SENTINEL", "DEBUG", -2, NONE, 0)
COMMAND(SENTINEL_FAILOVER, "SENTINEL", "FAILOVER", 3, NONE, 0)
COMMAND(SENTINEL_FLUSHCONFIG, "SENTINEL", "FLUSHCONFIG", 2, NONE, 0)
Expand Down Expand Up @@ -318,6 +320,7 @@ COMMAND(UNLINK, "UNLINK", NULL, -2, INDEX, 1)
COMMAND(UNSUBSCRIBE, "UNSUBSCRIBE", NULL, -1, NONE, 0)
COMMAND(UNWATCH, "UNWATCH", NULL, 1, NONE, 0)
COMMAND(WAIT, "WAIT", NULL, 3, NONE, 0)
COMMAND(WAITAOF, "WAITAOF", NULL, 4, NONE, 0)
COMMAND(WATCH, "WATCH", NULL, -2, INDEX, 1)
COMMAND(XACK, "XACK", NULL, -4, INDEX, 1)
COMMAND(XADD, "XADD", NULL, -5, INDEX, 1)
Expand Down
83 changes: 56 additions & 27 deletions gencommands.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,19 @@

# This script generates cmddef.h from the JSON files in the Redis repo
# describing the commands. This is done manually when commands have been added
# to Redis.
# to Redis or when you want add more commands implemented in modules, etc.
#
# Usage: ./gencommands.py path/to/redis/src/commands/*.json > cmddef.h
#
# Additional JSON files can be added to define custom commands. The JSON file
# format is not fully documented but hopefully the format can be understood from
# reading the existing JSON files. Alternatively, you can read the source code
# of this script to see what it does.
# Alternatively, the output of the script utils/generate-commands-json.py (which
# fetches the command metadata from a running Redis node) or the file
# commands.json from the redis-doc repo can be used as input to this script:
# https://github.com/redis/redis-doc/blob/master/commands.json
#
# Additional JSON files can be added to extend support for custom commands. The
# JSON file format is not fully documented but hopefully the format can be
# understood from reading the existing JSON files. Alternatively, you can read
# the source code of this script to see what it does.
#
# The key specifications part is documented here:
# https://redis.io/docs/reference/key-specs/
Expand All @@ -31,6 +36,15 @@
import sys
import re

# Returns True if any of the nested arguments is a key; False otherwise.
def any_argument_is_key(arguments):
for arg in arguments:
if arg.get("type") == "key":
return True
if "arguments" in arg and any_argument_is_key(arg["arguments"]):
return True
return False

# Returns a tuple (method, index) where method is one of the following:
#
# NONE = No keys
Expand All @@ -43,29 +57,22 @@
# keys (example EVAL)
def firstkey(props):
if not "key_specs" in props:
# Key specs missing. Best-effort fallback to "arguments" for modules. To
# avoid returning UNKNOWN instead of NONE for official Redis commands
# without keys, we check for "arity" which is always defined in Redis
# but not in the Redis Stack modules which also lack key specs.
if "arguments" in props and "arity" not in props:
# Key specs missing. Best-effort fallback to "arguments".
if "arguments" in props:
args = props["arguments"]
for i in range(1, len(args)):
arg = args[i - 1]
if not "type" in arg:
return ("NONE", 0)
if arg["type"] == "key":
if arg.get("type") == "key":
return ("INDEX", i)
elif arg["type"] == "string":
if "name" in arg and arg["name"] == "key":
# add-hoc case for RediSearch
return ("INDEX", i)
if "optional" in arg and arg["optional"]:
return ("UNKNOWN", 0)
if "multiple" in arg and arg["multiple"]:
return ("UNKNOWN", 0)
else:
elif arg.get("type") == "string" and arg.get("name") == "key":
# add-hoc case for RediSearch
return ("INDEX", i)
elif arg.get("optional") or arg.get("multiple") or "arguments" in arg:
# Too complex for this fallback.
return ("UNKNOWN", 0)
if any_argument_is_key(args):
return ("UNKNOWN", 0)
else:
return ("NONE", 0)
return ("NONE", 0)

if len(props["key_specs"]) == 0:
Expand All @@ -75,18 +82,39 @@ def firstkey(props):
# Otherwise we return -1 for unknown (for example if the first key is
# indicated by a keyword like KEYS or STREAMS).
begin_search = props["key_specs"][0]["begin_search"]
if not "index" in begin_search:
if "index" in begin_search:
# Redis source JSON files have this syntax
pos = begin_search["index"]["pos"]
elif begin_search.get("type") == "index" and "spec" in begin_search:
# generate-commands-json.py returns this syntax
pos = begin_search["spec"]["index"]
else:
return ("UNKNOWN", 0)
pos = begin_search["index"]["pos"]

find_keys = props["key_specs"][0]["find_keys"]
if "range" in find_keys:
if "range" in find_keys or find_keys.get("type") == "range":
# The first key is the arg at index pos.
# Redis source JSON files have this syntax:
# "find_keys": {
# "range": {...}
# }
# generate-commands-json.py returns this syntax:
# "find_keys": {
# "type": "range",
# "spec": {...}
# },
return ("INDEX", pos)
elif "keynum" in find_keys:
# The arg at pos is the number of keys and the next arg is the first key
# Redis source JSON files have this syntax
assert find_keys["keynum"]["keynumidx"] == 0
assert find_keys["keynum"]["firstkey"] == 1
return ("KEYNUM", pos)
elif find_keys.get("type") == "keynum":
# generate-commands-json.py returns this syntax
assert find_keys["spec"]["keynumidx"] == 0
assert find_keys["spec"]["firstkey"] == 1
return ("KEYNUM", pos)
else:
return ("UNKNOWN", 0)

Expand All @@ -105,7 +133,8 @@ def extract_command_info(name, props):
tokens = name.split(maxsplit=1)
if len(tokens) > 1:
name, subcommand = tokens
if firstkeypos > 0:
if firstkeypos > 0 and not "key_specs" in props:
# Position was inferred from "arguments"
firstkeypos += 1

arity = props["arity"] if "arity" in props else -1
Expand Down

0 comments on commit 88f9487

Please sign in to comment.