Skip to content

Commit a12c9af

Browse files
radeksimkoradeksimko
authored
indexer/walker: Avoid running jobs where not needed (#1006)
* indexer cleanup: Replace Defer with DependsOn * Introduce StateIgnore & plumb context through * state: Avoid deduplicating jobs on enqueuing * indexer: use IgnoreState * ci: bump benchmarks total timeout * update benchmarks thresholds * indexer: run ObtainSchema even if scheduling of an upstream job fails
1 parent aa15239 commit a12c9af

14 files changed

+364
-177
lines changed

‎.github/gobenchdata-checks.yml

+10-10
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,21 @@ checks:
44
benchmarks: [BenchmarkInitializeFolder_basic/local-single-module-no-provider]
55
diff: current.NsPerOp / 1000000 # ms
66
thresholds:
7-
min: 25
8-
max: 55
7+
min: 3
8+
max: 20
99
- package: ./internal/langserver/handlers
1010
name: local-single-submodule-no-provider
1111
benchmarks: [BenchmarkInitializeFolder_basic/local-single-submodule-no-provider]
1212
diff: current.NsPerOp / 1000000 # ms
1313
thresholds:
14-
min: 140
15-
max: 310
14+
min: 100
15+
max: 250
1616
- package: ./internal/langserver/handlers
1717
name: local-single-module-random
1818
benchmarks: [BenchmarkInitializeFolder_basic/local-single-module-random]
1919
diff: current.NsPerOp / 1000000 # ms
2020
thresholds:
21-
min: 140
21+
min: 100
2222
max: 300
2323
- package: ./internal/langserver/handlers
2424
name: local-single-module-aws
@@ -47,35 +47,35 @@ checks:
4747
diff: current.NsPerOp / 1000000 # ms
4848
thresholds:
4949
min: 1400
50-
max: 2200
50+
max: 5000
5151
- package: ./internal/langserver/handlers
5252
name: google-project
5353
benchmarks: [BenchmarkInitializeFolder_basic/google-project]
5454
diff: current.NsPerOp / 1000000 # ms
5555
thresholds:
5656
min: 1570
57-
max: 2450
57+
max: 9000
5858
- package: ./internal/langserver/handlers
5959
name: google-network
6060
benchmarks: [BenchmarkInitializeFolder_basic/google-network]
6161
diff: current.NsPerOp / 1000000 # ms
6262
thresholds:
6363
min: 1430
64-
max: 2700
64+
max: 15000
6565
- package: ./internal/langserver/handlers
6666
name: google-gke
6767
benchmarks: [BenchmarkInitializeFolder_basic/google-gke]
6868
diff: current.NsPerOp / 1000000 # ms
6969
thresholds:
7070
min: 1500
71-
max: 5100
71+
max: 20000
7272
- package: ./internal/langserver/handlers
7373
name: k8s-metrics-server
7474
benchmarks: [BenchmarkInitializeFolder_basic/k8s-metrics-server]
7575
diff: current.NsPerOp / 1000000 # ms
7676
thresholds:
7777
min: 1000
78-
max: 3200
78+
max: 4000
7979
- package: ./internal/langserver/handlers
8080
name: k8s-dashboard
8181
benchmarks: [BenchmarkInitializeFolder_basic/k8s-dashboard]

‎.github/workflows/benchmarks.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
-bench=InitializeFolder_basic \
4242
-run=^# \
4343
-benchtime=60s \
44-
-timeout=30m | tee ${{ runner.temp }}/benchmarks.txt
44+
-timeout=60m | tee ${{ runner.temp }}/benchmarks.txt
4545
-
4646
name: Evaluate benchmarks
4747
id: bench-eval

‎internal/indexer/document_change.go

+21-15
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,17 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
1515
parseId, err := idx.jobStore.EnqueueJob(job.Job{
1616
Dir: modHandle,
1717
Func: func(ctx context.Context) error {
18-
return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
18+
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
1919
},
20-
Type: op.OpTypeParseModuleConfiguration.String(),
20+
Type: op.OpTypeParseModuleConfiguration.String(),
21+
IgnoreState: true,
2122
})
2223
if err != nil {
2324
return ids, err
2425
}
2526
ids = append(ids, parseId)
2627

27-
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId})
28+
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true)
2829
if err != nil {
2930
return ids, err
3031
}
@@ -33,9 +34,10 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
3334
parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{
3435
Dir: modHandle,
3536
Func: func(ctx context.Context) error {
36-
return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
37+
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
3738
},
38-
Type: op.OpTypeParseVariables.String(),
39+
Type: op.OpTypeParseVariables.String(),
40+
IgnoreState: true,
3941
})
4042
if err != nil {
4143
return ids, err
@@ -47,8 +49,9 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
4749
Func: func(ctx context.Context) error {
4850
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
4951
},
50-
Type: op.OpTypeDecodeVarsReferences.String(),
51-
DependsOn: job.IDs{parseVarsId},
52+
Type: op.OpTypeDecodeVarsReferences.String(),
53+
DependsOn: job.IDs{parseVarsId},
54+
IgnoreState: true,
5255
})
5356
if err != nil {
5457
return ids, err
@@ -58,16 +61,17 @@ func (idx *Indexer) DocumentChanged(modHandle document.DirHandle) (job.IDs, erro
5861
return ids, nil
5962
}
6063

61-
func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) {
64+
func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) {
6265
ids := make(job.IDs, 0)
6366

6467
metaId, err := idx.jobStore.EnqueueJob(job.Job{
6568
Dir: modHandle,
6669
Func: func(ctx context.Context) error {
67-
return module.LoadModuleMetadata(idx.modStore, modHandle.Path())
70+
return module.LoadModuleMetadata(ctx, idx.modStore, modHandle.Path())
6871
},
69-
Type: op.OpTypeLoadModuleMetadata.String(),
70-
DependsOn: dependsOn,
72+
Type: op.OpTypeLoadModuleMetadata.String(),
73+
DependsOn: dependsOn,
74+
IgnoreState: ignoreState,
7175
})
7276
if err != nil {
7377
return ids, err
@@ -79,8 +83,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs
7983
Func: func(ctx context.Context) error {
8084
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
8185
},
82-
Type: op.OpTypeDecodeReferenceTargets.String(),
83-
DependsOn: job.IDs{metaId},
86+
Type: op.OpTypeDecodeReferenceTargets.String(),
87+
DependsOn: job.IDs{metaId},
88+
IgnoreState: ignoreState,
8489
})
8590
if err != nil {
8691
return ids, err
@@ -92,8 +97,9 @@ func (idx *Indexer) decodeModule(modHandle document.DirHandle, dependsOn job.IDs
9297
Func: func(ctx context.Context) error {
9398
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
9499
},
95-
Type: op.OpTypeDecodeReferenceOrigins.String(),
96-
DependsOn: job.IDs{metaId},
100+
Type: op.OpTypeDecodeReferenceOrigins.String(),
101+
DependsOn: job.IDs{metaId},
102+
IgnoreState: ignoreState,
97103
})
98104
if err != nil {
99105
return ids, err

‎internal/indexer/document_open.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,17 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error
4040
parseId, err := idx.jobStore.EnqueueJob(job.Job{
4141
Dir: modHandle,
4242
Func: func(ctx context.Context) error {
43-
return module.ParseModuleConfiguration(idx.fs, idx.modStore, modHandle.Path())
43+
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, modHandle.Path())
4444
},
45-
Type: op.OpTypeParseModuleConfiguration.String(),
45+
Type: op.OpTypeParseModuleConfiguration.String(),
46+
IgnoreState: true,
4647
})
4748
if err != nil {
4849
return ids, err
4950
}
5051
ids = append(ids, parseId)
5152

52-
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId})
53+
modIds, err := idx.decodeModule(modHandle, job.IDs{parseId}, true)
5354
if err != nil {
5455
return ids, err
5556
}
@@ -58,9 +59,10 @@ func (idx *Indexer) DocumentOpened(modHandle document.DirHandle) (job.IDs, error
5859
parseVarsId, err := idx.jobStore.EnqueueJob(job.Job{
5960
Dir: modHandle,
6061
Func: func(ctx context.Context) error {
61-
return module.ParseVariables(idx.fs, idx.modStore, modHandle.Path())
62+
return module.ParseVariables(ctx, idx.fs, idx.modStore, modHandle.Path())
6263
},
63-
Type: op.OpTypeParseVariables.String(),
64+
Type: op.OpTypeParseVariables.String(),
65+
IgnoreState: true,
6466
})
6567
if err != nil {
6668
return ids, err

‎internal/indexer/module_calls.go

+21-15
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
op "github.com/hashicorp/terraform-ls/internal/terraform/module/operation"
1212
)
1313

14-
func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (job.IDs, error) {
14+
func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle, ignoreState bool) (job.IDs, error) {
1515
jobIds := make(job.IDs, 0)
1616

1717
moduleCalls, err := idx.modStore.ModuleCalls(modHandle.Path())
@@ -43,9 +43,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
4343
parseId, err := idx.jobStore.EnqueueJob(job.Job{
4444
Dir: mcHandle,
4545
Func: func(ctx context.Context) error {
46-
return module.ParseModuleConfiguration(idx.fs, idx.modStore, mcPath)
46+
return module.ParseModuleConfiguration(ctx, idx.fs, idx.modStore, mcPath)
4747
},
48-
Type: op.OpTypeParseModuleConfiguration.String(),
48+
Type: op.OpTypeParseModuleConfiguration.String(),
49+
IgnoreState: ignoreState,
4950
})
5051
if err != nil {
5152
multierror.Append(errs, err)
@@ -60,9 +61,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
6061
Dir: mcHandle,
6162
Type: op.OpTypeLoadModuleMetadata.String(),
6263
Func: func(ctx context.Context) error {
63-
return module.LoadModuleMetadata(idx.modStore, mcPath)
64+
return module.LoadModuleMetadata(ctx, idx.modStore, mcPath)
6465
},
65-
DependsOn: job.IDs{parseId},
66+
DependsOn: job.IDs{parseId},
67+
IgnoreState: ignoreState,
6668
})
6769
if err != nil {
6870
multierror.Append(errs, err)
@@ -73,7 +75,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
7375
}
7476

7577
if parseId != "" {
76-
ids, err := idx.collectReferences(mcHandle, refCollectionDeps)
78+
ids, err := idx.collectReferences(mcHandle, refCollectionDeps, ignoreState)
7779
if err != nil {
7880
multierror.Append(errs, err)
7981
} else {
@@ -84,9 +86,10 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
8486
varsParseId, err := idx.jobStore.EnqueueJob(job.Job{
8587
Dir: mcHandle,
8688
Func: func(ctx context.Context) error {
87-
return module.ParseVariables(idx.fs, idx.modStore, mcPath)
89+
return module.ParseVariables(ctx, idx.fs, idx.modStore, mcPath)
8890
},
89-
Type: op.OpTypeParseVariables.String(),
91+
Type: op.OpTypeParseVariables.String(),
92+
IgnoreState: ignoreState,
9093
})
9194
if err != nil {
9295
multierror.Append(errs, err)
@@ -100,8 +103,9 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
100103
Func: func(ctx context.Context) error {
101104
return module.DecodeVarsReferences(ctx, idx.modStore, idx.schemaStore, mcPath)
102105
},
103-
Type: op.OpTypeDecodeVarsReferences.String(),
104-
DependsOn: job.IDs{varsParseId},
106+
Type: op.OpTypeDecodeVarsReferences.String(),
107+
DependsOn: job.IDs{varsParseId},
108+
IgnoreState: ignoreState,
105109
})
106110
if err != nil {
107111
multierror.Append(errs, err)
@@ -114,7 +118,7 @@ func (idx *Indexer) decodeInstalledModuleCalls(modHandle document.DirHandle) (jo
114118
return jobIds, errs.ErrorOrNil()
115119
}
116120

117-
func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs) (job.IDs, error) {
121+
func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn job.IDs, ignoreState bool) (job.IDs, error) {
118122
ids := make(job.IDs, 0)
119123

120124
var errs *multierror.Error
@@ -124,8 +128,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo
124128
Func: func(ctx context.Context) error {
125129
return module.DecodeReferenceTargets(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
126130
},
127-
Type: op.OpTypeDecodeReferenceTargets.String(),
128-
DependsOn: dependsOn,
131+
Type: op.OpTypeDecodeReferenceTargets.String(),
132+
DependsOn: dependsOn,
133+
IgnoreState: ignoreState,
129134
})
130135
if err != nil {
131136
errs = multierror.Append(errs, err)
@@ -138,8 +143,9 @@ func (idx *Indexer) collectReferences(modHandle document.DirHandle, dependsOn jo
138143
Func: func(ctx context.Context) error {
139144
return module.DecodeReferenceOrigins(ctx, idx.modStore, idx.schemaStore, modHandle.Path())
140145
},
141-
Type: op.OpTypeDecodeReferenceOrigins.String(),
142-
DependsOn: dependsOn,
146+
Type: op.OpTypeDecodeReferenceOrigins.String(),
147+
DependsOn: dependsOn,
148+
IgnoreState: ignoreState,
143149
})
144150
if err != nil {
145151
errs = multierror.Append(errs, err)

0 commit comments

Comments
 (0)