Skip to content

Commit 4b8a138

Browse files
srenatussrenatus
authored
topdown+rego+server: allow opt-in for evaluating non-det builtins in PE (#7313)
* topdown+rego: allow opt-in for evaluating non-det builtins in PE Some use cases of PE, notably generating queries that are to be translated into filters of some sort (think SQL), require the evaluation of non-deterministic builtins. This is because the result of the builtin informs what queries are returned. Imagine that the user associated with a request is known at PE-time, but we need extra information from an HTTP API to determine the filters that should be applied. Previously, that was just impossible to do. Now, we can opt-in to evaluate non-det builtins during PE from the Rego API. Note that it would probably make sense to include this in the inlining controls, as sent to the Compile API. (Considered out of scope for this PR.) Also note that this will take highest precedence over the `ast.IgnoreDuringPartialEval` map and the "Nondeterministic" value of the registered builtin. If the new option is provided, both of these are ignored. Signed-off-by: Stephan Renatus <stephan@styra.com> * server+rego: expose nondeterministicBuiltins via inlining controls With `foo.rego` as ```rego package ex include if input.fruits.name == object.get(http.send(input.req).body, input.path, "unknown") ``` the following queries show the difference: ```interactive $ curl -v http://127.0.0.1:8181/v1/compile \ -d '{"input": {"req": {"url": "https://httpbin.org/json", "method":"GET"}, "path": ["slideshow", "title"]}, "query": "data.ex.include", "unknowns": ["input.fruits"]}' { "result": { "queries": [ [ { "index": 0, "terms": [ { "type": "ref", "value": [ { "type": "var", "value": "http" }, { "type": "string", "value": "send" } ] }, { "type": "object", "value": [ [ { "type": "string", "value": "method" }, { "type": "string", "value": "GET" } ], [ { "type": "string", "value": "url" }, { "type": "string", "value": "https://httpbin.org/json" } ] ] }, { "type": "var", "value": "__local0__1" } ] }, { "index": 1, "terms": [ { "type": "ref", "value": [ { "type": "var", "value": "eq" } ] }, { "type": "ref", "value": [ { "type": "var", "value": "input" }, { "type": "string", "value": "fruits" }, { "type": "string", "value": "name" } ] }, { "type": "call", "value": [ { "type": "ref", "value": [ { "type": "var", "value": "object" }, { "type": "string", "value": "get" } ] }, { "type": "ref", "value": [ { "type": "var", "value": "__local0__1" }, { "type": "string", "value": "body" } ] }, { "type": "array", "value": [ { "type": "string", "value": "slideshow" }, { "type": "string", "value": "title" } ] }, { "type": "string", "value": "unknown" } ] } ] } ] ] } } ``` Here, the builtin call to http.send is preserved. If we also pass `nondeterminsticBuiltins: true` to the options, we get this: ```interactive $ curl http://127.0.0.1:8181/v1/compile \ -d '{"input": {"req": {"url": "https://httpbin.org/json", "method":"GET"}, "path": ["slideshow", "title"]}, "query": "data.ex.include", "unknowns": ["input.fruits"], "options": {"nondeterministicBuiltins": true}}' { "result": { "queries": [ [ { "index": 0, "terms": [ { "type": "ref", "value": [ { "type": "var", "value": "eq" } ] }, { "type": "ref", "value": [ { "type": "var", "value": "input" }, { "type": "string", "value": "fruits" }, { "type": "string", "value": "name" } ] }, { "type": "string", "value": "Sample Slide Show" } ] } ] ] } } ``` Here, all args to http.send have been known at PE time and the call was fully evaluated. Signed-off-by: Stephan Renatus <stephan@styra.com> * cmd/eval: expose --nondeterminstic-builtins for new PE control ```interactive $ echo '{"req": {"url": "https://httpbin.org/json", "method":"GET"}, "path": ["slideshow", "title"]}'| ./opa_darwin_amd64 eval -fpretty -p -I -d foo.rego -u input.fruits data.ex.include +---------+-------------------------------------------------------------------------------------+ | Query 1 | http.send({"method": "GET", "url": "https://httpbin.org/json"}, __local0__1) | | | input.fruits.name = object.get(__local0__1.body, ["slideshow", "title"], "unknown") | +---------+-------------------------------------------------------------------------------------+ $ echo '{"req": {"url": "https://httpbin.org/json", "method":"GET"}, "path": ["slideshow", "title"]}'| ./opa_darwin_amd64 eval -fpretty -p -I -d foo.rego -u input.fruits data.ex.include --nondeterminstic-builtins +---------+-----------------------------------------+ | Query 1 | input.fruits.name = "Sample Slide Show" | +---------+-----------------------------------------+ ``` Signed-off-by: Stephan Renatus <stephan@styra.com> --------- Signed-off-by: Stephan Renatus <stephan@styra.com>
1 parent 50a8c96 commit 4b8a138

File tree

8 files changed

+190
-108
lines changed

8 files changed

+190
-108
lines changed

‎cmd/eval.go

+42-39
Original file line numberDiff line numberDiff line change
@@ -40,45 +40,46 @@ var (
4040
)
4141

4242
type evalCommandParams struct {
43-
capabilities *capabilitiesFlag
44-
coverage bool
45-
partial bool
46-
unknowns []string
47-
disableInlining []string
48-
shallowInlining bool
49-
disableIndexing bool
50-
disableEarlyExit bool
51-
strictBuiltinErrors bool
52-
showBuiltinErrors bool
53-
dataPaths repeatedStringFlag
54-
inputPath string
55-
imports repeatedStringFlag
56-
pkg string
57-
stdin bool
58-
stdinInput bool
59-
explain *util.EnumFlag
60-
metrics bool
61-
instrument bool
62-
ignore []string
63-
outputFormat *util.EnumFlag
64-
profile bool
65-
profileCriteria repeatedStringFlag
66-
profileLimit intFlag
67-
count int
68-
prettyLimit intFlag
69-
fail bool
70-
failDefined bool
71-
bundlePaths repeatedStringFlag
72-
schema *schemaFlags
73-
target *util.EnumFlag
74-
timeout time.Duration
75-
optimizationLevel int
76-
entrypoints repeatedStringFlag
77-
strict bool
78-
v0Compatible bool
79-
v1Compatible bool
80-
traceVarValues bool
81-
ReadAstValuesFromStore bool
43+
capabilities *capabilitiesFlag
44+
coverage bool
45+
partial bool
46+
unknowns []string
47+
disableInlining []string
48+
nondeterministicBuiltions bool
49+
shallowInlining bool
50+
disableIndexing bool
51+
disableEarlyExit bool
52+
strictBuiltinErrors bool
53+
showBuiltinErrors bool
54+
dataPaths repeatedStringFlag
55+
inputPath string
56+
imports repeatedStringFlag
57+
pkg string
58+
stdin bool
59+
stdinInput bool
60+
explain *util.EnumFlag
61+
metrics bool
62+
instrument bool
63+
ignore []string
64+
outputFormat *util.EnumFlag
65+
profile bool
66+
profileCriteria repeatedStringFlag
67+
profileLimit intFlag
68+
count int
69+
prettyLimit intFlag
70+
fail bool
71+
failDefined bool
72+
bundlePaths repeatedStringFlag
73+
schema *schemaFlags
74+
target *util.EnumFlag
75+
timeout time.Duration
76+
optimizationLevel int
77+
entrypoints repeatedStringFlag
78+
strict bool
79+
v0Compatible bool
80+
v1Compatible bool
81+
traceVarValues bool
82+
ReadAstValuesFromStore bool
8283
}
8384

8485
func (p *evalCommandParams) regoVersion() ast.RegoVersion {
@@ -328,6 +329,7 @@ access.
328329
evalCommand.Flags().BoolVarP(&params.coverage, "coverage", "", false, "report coverage")
329330
evalCommand.Flags().StringArrayVarP(&params.disableInlining, "disable-inlining", "", []string{}, "set paths of documents to exclude from inlining")
330331
evalCommand.Flags().BoolVarP(&params.shallowInlining, "shallow-inlining", "", false, "disable inlining of rules that depend on unknowns")
332+
evalCommand.Flags().BoolVarP(&params.nondeterministicBuiltions, "nondeterminstic-builtins", "", false, "evaluate nondeterministic builtins (if all arguments are known) during partial eval")
331333
evalCommand.Flags().BoolVar(&params.disableIndexing, "disable-indexing", false, "disable indexing optimizations")
332334
evalCommand.Flags().BoolVar(&params.disableEarlyExit, "disable-early-exit", false, "disable 'early exit' optimizations")
333335
evalCommand.Flags().BoolVarP(&params.strictBuiltinErrors, "strict-builtin-errors", "", false, "treat the first built-in function error encountered as fatal")
@@ -577,6 +579,7 @@ func setupEval(args []string, params evalCommandParams) (*evalContext, error) {
577579
evalArgs := []rego.EvalOption{
578580
rego.EvalRuleIndexing(!params.disableIndexing),
579581
rego.EvalEarlyExit(!params.disableEarlyExit),
582+
rego.EvalNondeterministicBuiltins(params.nondeterministicBuiltions),
580583
}
581584

582585
if len(params.imports.v) > 0 {

‎docs/content/rest-api.md

+6-1
Original file line numberDiff line numberDiff line change
@@ -1320,6 +1320,11 @@ on the OPA blog shows how SQL can be generated based on Compile API output.
13201320
For more details on Partial Evaluation in OPA, please refer to
13211321
[this blog post](https://blog.openpolicyagent.org/partial-evaluation-162750eaf422).
13221322

1323+
Note that nondeterminstic builtins (like `http.send`) are _not evaluated_ during PE.
1324+
You can change that by providing `nondeterminsticBuiltins: true` in your payload options.
1325+
This would be desirable when using PE for generating filters using extra information
1326+
from `http.send`.
1327+
13231328
#### Request Body
13241329

13251330
Compile API requests contain the following fields:
@@ -1328,7 +1333,7 @@ Compile API requests contain the following fields:
13281333
| --- | --- | --- | --- |
13291334
| `query` | `string` | Yes | The query to partially evaluate and compile. |
13301335
| `input` | `any` | No | The input document to use during partial evaluation (default: undefined). |
1331-
| `options` | `object[string, any]` | No | Additional options to use during partial evaluation. Only `disableInlining` option is supported. (default: undefined). |
1336+
| `options` | `object[string, any]` | No | Additional options to use during partial evaluation: `disableInlining` (default: undefined) and `nondeterminsticBuiltins` (default: false). |
13321337
| `unknowns` | `array[string]` | No | The terms to treat as unknown during partial evaluation (default: `["input"]`]). |
13331338

13341339
### Request Headers

‎v1/rego/rego.go

+52-29
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ type EvalContext struct {
113113
compiledQuery compiledQuery
114114
unknowns []string
115115
disableInlining []ast.Ref
116+
nondeterministicBuiltins bool
116117
parsedUnknowns []*ast.Term
117118
indexing bool
118119
earlyExit bool
@@ -372,6 +373,15 @@ func EvalVirtualCache(vc topdown.VirtualCache) EvalOption {
372373
}
373374
}
374375

376+
// EvalNondeterministicBuiltins causes non-deterministic builtins to be evalued
377+
// during partial evaluation. This is needed to pull in external data, or validate
378+
// a JWT, during PE, so that the result informs what queries are returned.
379+
func EvalNondeterministicBuiltins(yes bool) EvalOption {
380+
return func(e *EvalContext) {
381+
e.nondeterministicBuiltins = yes
382+
}
383+
}
384+
375385
func (pq preparedQuery) Modules() map[string]*ast.Module {
376386
mods := make(map[string]*ast.Module)
377387

@@ -394,24 +404,25 @@ func (pq preparedQuery) Modules() map[string]*ast.Module {
394404
// been opened.
395405
func (pq preparedQuery) newEvalContext(ctx context.Context, options []EvalOption) (*EvalContext, func(context.Context), error) {
396406
ectx := &EvalContext{
397-
hasInput: false,
398-
rawInput: nil,
399-
parsedInput: nil,
400-
metrics: nil,
401-
txn: nil,
402-
instrument: false,
403-
instrumentation: nil,
404-
partialNamespace: pq.r.partialNamespace,
405-
queryTracers: nil,
406-
unknowns: pq.r.unknowns,
407-
parsedUnknowns: pq.r.parsedUnknowns,
408-
compiledQuery: compiledQuery{},
409-
indexing: true,
410-
earlyExit: true,
411-
resolvers: pq.r.resolvers,
412-
printHook: pq.r.printHook,
413-
capabilities: pq.r.capabilities,
414-
strictBuiltinErrors: pq.r.strictBuiltinErrors,
407+
hasInput: false,
408+
rawInput: nil,
409+
parsedInput: nil,
410+
metrics: nil,
411+
txn: nil,
412+
instrument: false,
413+
instrumentation: nil,
414+
partialNamespace: pq.r.partialNamespace,
415+
queryTracers: nil,
416+
unknowns: pq.r.unknowns,
417+
parsedUnknowns: pq.r.parsedUnknowns,
418+
nondeterministicBuiltins: pq.r.nondeterministicBuiltins,
419+
compiledQuery: compiledQuery{},
420+
indexing: true,
421+
earlyExit: true,
422+
resolvers: pq.r.resolvers,
423+
printHook: pq.r.printHook,
424+
capabilities: pq.r.capabilities,
425+
strictBuiltinErrors: pq.r.strictBuiltinErrors,
415426
}
416427

417428
for _, o := range options {
@@ -580,6 +591,7 @@ type Rego struct {
580591
parsedUnknowns []*ast.Term
581592
disableInlining []string
582593
shallowInlining bool
594+
nondeterministicBuiltins bool
583595
skipPartialNamespace bool
584596
partialNamespace string
585597
modules []rawModule
@@ -922,6 +934,15 @@ func DisableInlining(paths []string) func(r *Rego) {
922934
}
923935
}
924936

937+
// NondeterministicBuiltins causes non-deterministic builtins to be evalued during
938+
// partial evaluation. This is needed to pull in external data, or validate a JWT,
939+
// during PE, so that the result informs what queries are returned.
940+
func NondeterministicBuiltins(yes bool) func(r *Rego) {
941+
return func(r *Rego) {
942+
r.nondeterministicBuiltins = yes
943+
}
944+
}
945+
925946
// ShallowInlining prevents rules that depend on unknown values from being inlined.
926947
// Rules that only depend on known values are inlined.
927948
func ShallowInlining(yes bool) func(r *Rego) {
@@ -2334,17 +2355,18 @@ func (r *Rego) partialResult(ctx context.Context, pCfg *PrepareConfig) (PartialR
23342355
}
23352356

23362357
ectx := &EvalContext{
2337-
parsedInput: r.parsedInput,
2338-
metrics: r.metrics,
2339-
txn: r.txn,
2340-
partialNamespace: r.partialNamespace,
2341-
queryTracers: r.queryTracers,
2342-
compiledQuery: r.compiledQueries[partialResultQueryType],
2343-
instrumentation: r.instrumentation,
2344-
indexing: true,
2345-
resolvers: r.resolvers,
2346-
capabilities: r.capabilities,
2347-
strictBuiltinErrors: r.strictBuiltinErrors,
2358+
parsedInput: r.parsedInput,
2359+
metrics: r.metrics,
2360+
txn: r.txn,
2361+
partialNamespace: r.partialNamespace,
2362+
queryTracers: r.queryTracers,
2363+
compiledQuery: r.compiledQueries[partialResultQueryType],
2364+
instrumentation: r.instrumentation,
2365+
indexing: true,
2366+
resolvers: r.resolvers,
2367+
capabilities: r.capabilities,
2368+
strictBuiltinErrors: r.strictBuiltinErrors,
2369+
nondeterministicBuiltins: r.nondeterministicBuiltins,
23482370
}
23492371

23502372
disableInlining := r.disableInlining
@@ -2441,6 +2463,7 @@ func (r *Rego) partial(ctx context.Context, ectx *EvalContext) (*PartialQueries,
24412463
WithInstrumentation(ectx.instrumentation).
24422464
WithUnknowns(unknowns).
24432465
WithDisableInlining(ectx.disableInlining).
2466+
WithNondeterministicBuiltins(ectx.nondeterministicBuiltins).
24442467
WithRuntime(r.runtime).
24452468
WithIndexing(ectx.indexing).
24462469
WithEarlyExit(ectx.earlyExit).

‎v1/server/server.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -1405,6 +1405,7 @@ func (s *Server) v1CompilePost(w http.ResponseWriter, r *http.Request) {
14051405
rego.ParsedInput(request.Input),
14061406
rego.ParsedUnknowns(request.Unknowns),
14071407
rego.DisableInlining(request.Options.DisableInlining),
1408+
rego.NondeterministicBuiltins(request.Options.NondeterminsiticBuiltins),
14081409
rego.QueryTracer(buf),
14091410
rego.Instrument(includeInstrumentation),
14101411
rego.Metrics(m),
@@ -2856,7 +2857,8 @@ type compileRequest struct {
28562857
}
28572858

28582859
type compileRequestOptions struct {
2859-
DisableInlining []string
2860+
DisableInlining []string
2861+
NondeterminsiticBuiltins bool
28602862
}
28612863

28622864
func readInputCompilePostV1(reqBytes []byte, queryParserOptions ast.ParserOptions) (*compileRequest, *types.ErrorV1) {
@@ -2898,16 +2900,15 @@ func readInputCompilePostV1(reqBytes []byte, queryParserOptions ast.ParserOption
28982900
}
28992901
}
29002902

2901-
result := &compileRequest{
2903+
return &compileRequest{
29022904
Query: query,
29032905
Input: input,
29042906
Unknowns: unknowns,
29052907
Options: compileRequestOptions{
2906-
DisableInlining: request.Options.DisableInlining,
2908+
DisableInlining: request.Options.DisableInlining,
2909+
NondeterminsiticBuiltins: request.Options.NondeterministicBuiltins,
29072910
},
2908-
}
2909-
2910-
return result, nil
2911+
}, nil
29112912
}
29122913

29132914
var indexHTML, _ = template.New("index").Parse(`

‎v1/server/types/types.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,8 @@ type CompileRequestV1 struct {
374374
Query string `json:"query"`
375375
Unknowns *[]string `json:"unknowns"`
376376
Options struct {
377-
DisableInlining []string `json:"disableInlining,omitempty"`
377+
DisableInlining []string `json:"disableInlining,omitempty"`
378+
NondeterministicBuiltins bool `json:"nondeterministicBuiltins"`
378379
} `json:"options,omitempty"`
379380
}
380381

‎v1/topdown/query.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ type Query struct {
4646
instr *Instrumentation
4747
disableInlining []ast.Ref
4848
shallowInlining bool
49+
nondeterministicBuiltins bool
4950
genvarprefix string
5051
runtime *ast.Term
5152
builtins map[string]*Builtin
@@ -313,6 +314,14 @@ func (q *Query) WithVirtualCache(vc VirtualCache) *Query {
313314
return q
314315
}
315316

317+
// WithNondeterministicBuiltins causes non-deterministic builtins to be evalued
318+
// during partial evaluation. This is needed to pull in external data, or validate
319+
// a JWT, during PE, so that the result informs what queries are returned.
320+
func (q *Query) WithNondeterministicBuiltins(yes bool) *Query {
321+
q.nondeterministicBuiltins = yes
322+
return q
323+
}
324+
316325
// PartialRun executes partial evaluation on the query with respect to unknown
317326
// values. Partial evaluation attempts to evaluate as much of the query as
318327
// possible without requiring values for the unknowns set on the query. The
@@ -380,7 +389,8 @@ func (q *Query) PartialRun(ctx context.Context) (partials []ast.Body, support []
380389
saveNamespace: ast.StringTerm(q.partialNamespace),
381390
skipSaveNamespace: q.skipSaveNamespace,
382391
inliningControl: &inliningControl{
383-
shallow: q.shallowInlining,
392+
shallow: q.shallowInlining,
393+
nondeterministicBuiltins: q.nondeterministicBuiltins,
384394
},
385395
genvarprefix: q.genvarprefix,
386396
runtime: q.runtime,

‎v1/topdown/save.go

+10-3
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,13 @@ func saveRequired(c *ast.Compiler, ic *inliningControl, icIgnoreInternal bool, s
365365
}
366366
switch node := node.(type) {
367367
case *ast.Expr:
368-
found = len(node.With) > 0 || ignoreExprDuringPartial(node)
368+
found = len(node.With) > 0
369+
if found {
370+
return found
371+
}
372+
if !ic.nondeterministicBuiltins { // skip evaluating non-det builtins for PE
373+
found = ignoreExprDuringPartial(node)
374+
}
369375
case *ast.Term:
370376
switch v := node.Value.(type) {
371377
case ast.Var:
@@ -422,8 +428,9 @@ func ignoreDuringPartial(bi *ast.Builtin) bool {
422428
}
423429

424430
type inliningControl struct {
425-
shallow bool
426-
disable []disableInliningFrame
431+
shallow bool
432+
disable []disableInliningFrame
433+
nondeterministicBuiltins bool // evaluate non-det builtins during PE (if args are known)
427434
}
428435

429436
type disableInliningFrame struct {

0 commit comments

Comments
 (0)