Skip to content

Commit e062486

Browse files
leebensonleebenson
authored
enhancement(observability): Vector tap, v2 (#6610)
* tap v2 * wip Signed-off-by: Lee Benson <lee@leebenson.com> * more wip Signed-off-by: Lee Benson <lee@leebenson.com> * two-way control Signed-off-by: Lee Benson <lee@leebenson.com> * add component name Signed-off-by: Lee Benson <lee@leebenson.com> * attach sans detach wip Signed-off-by: Lee Benson <lee@leebenson.com> * fix atomic Signed-off-by: Lee Benson <lee@leebenson.com> * survive reloads * componentNames Signed-off-by: Lee Benson <lee@leebenson.com> * persistent subscription Signed-off-by: Lee Benson <lee@leebenson.com> * component not matched Signed-off-by: Lee Benson <lee@leebenson.com> * sample Signed-off-by: Lee Benson <lee@leebenson.com> * head/tail/range Signed-off-by: Lee Benson <lee@leebenson.com> * wip * refactor for limit Signed-off-by: Lee Benson <lee@leebenson.com> * bounded chan Signed-off-by: Lee Benson <lee@leebenson.com> * await Signed-off-by: Lee Benson <lee@leebenson.com> * fix api tests Signed-off-by: Lee Benson <lee@leebenson.com> * typos, etc Signed-off-by: Lee Benson <lee@leebenson.com> * period Signed-off-by: Lee Benson <lee@leebenson.com> * period Signed-off-by: Lee Benson <lee@leebenson.com> * some tests Signed-off-by: Lee Benson <lee@leebenson.com> * wildcards Signed-off-by: Lee Benson <lee@leebenson.com> * comments Signed-off-by: Lee Benson <lee@leebenson.com> * diff Signed-off-by: Lee Benson <lee@leebenson.com> * remove from topology Signed-off-by: Lee Benson <lee@leebenson.com> * tweaks Signed-off-by: Lee Benson <lee@leebenson.com> * param swap Signed-off-by: Lee Benson <lee@leebenson.com> * combine use Signed-off-by: Lee Benson <lee@leebenson.com> * dedupe Signed-off-by: Lee Benson <lee@leebenson.com> * clean up Signed-off-by: Lee Benson <lee@leebenson.com> * -inspect Signed-off-by: Lee Benson <lee@leebenson.com> * merge Signed-off-by: Lee Benson <lee@leebenson.com> * improvements Signed-off-by: Lee Benson <lee@leebenson.com> * clippy Signed-off-by: Lee Benson <lee@leebenson.com> * Algorithm R Signed-off-by: Lee Benson <lee@leebenson.com> * fix batch Signed-off-by: Lee Benson <lee@leebenson.com> * comments Signed-off-by: Lee Benson <lee@leebenson.com> * simplify Signed-off-by: Lee Benson <lee@leebenson.com> * commentary Signed-off-by: Lee Benson <lee@leebenson.com> * TapResult -> TapPayload Signed-off-by: Lee Benson <lee@leebenson.com> * debug * clean up Signed-off-by: Lee Benson <lee@leebenson.com> * tests Signed-off-by: Lee Benson <lee@leebenson.com> * tweaks Signed-off-by: Lee Benson <lee@leebenson.com> * happy clippy Signed-off-by: Lee Benson <lee@leebenson.com> * fix reload Signed-off-by: Lee Benson <lee@leebenson.com> * graphql schema Signed-off-by: Lee Benson <lee@leebenson.com> * debug Signed-off-by: Lee Benson <lee@leebenson.com> * SendError Signed-off-by: Lee Benson <lee@leebenson.com> * check-fmt Signed-off-by: Lee Benson <lee@leebenson.com> * infallible log string Signed-off-by: Lee Benson <lee@leebenson.com> * gen schema Signed-off-by: Lee Benson <lee@leebenson.com> * debug Signed-off-by: Lee Benson <lee@leebenson.com> * usize sort Signed-off-by: Lee Benson <lee@leebenson.com> * event union Signed-off-by: Lee Benson <lee@leebenson.com> * event encoding Signed-off-by: Lee Benson <lee@leebenson.com> * commentary Signed-off-by: Lee Benson <lee@leebenson.com> * schema update Signed-off-by: Lee Benson <lee@leebenson.com> * payload union Signed-off-by: Lee Benson <lee@leebenson.com> * schema Signed-off-by: Lee Benson <lee@leebenson.com> * fix shutdown Signed-off-by: Lee Benson <lee@leebenson.com> * -EventEncodingType Signed-off-by: Lee Benson <lee@leebenson.com> * TapSink sink Signed-off-by: Lee Benson <lee@leebenson.com> * SinkExt in test Signed-off-by: Lee Benson <lee@leebenson.com> * update for tokio 1.x; non-blocking Signed-off-by: Lee Benson <lee@leebenson.com> * fix test Signed-off-by: Lee Benson <lee@leebenson.com>
1 parent 1157d51 commit e062486

File tree

16 files changed

+1107
-104
lines changed

16 files changed

+1107
-104
lines changed

‎Cargo.lock

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+3-2
Original file line numberDiff line numberDiff line change
@@ -151,8 +151,8 @@ gouth = { version = "0.1", optional = true }
151151
smpl_jwt = { version = "0.6.1", optional = true }
152152

153153
# API
154-
async-graphql = { version = "2.5.9", optional = true }
155-
async-graphql-warp = { version = "2.5.9", optional = true }
154+
async-graphql = { version = "=2.5.9", optional = true }
155+
async-graphql-warp = { version = "=2.5.9", optional = true }
156156
itertools = { version = "0.10.0", optional = true }
157157

158158
# API client
@@ -243,6 +243,7 @@ twox-hash = "1.6"
243243
url = "2.2.1"
244244
uuid = { version = "0.8", features = ["serde", "v4"], optional = true }
245245
warp = { version = "0.3.0", default-features = false, optional = true }
246+
cfg-if = "1.0.0"
246247

247248
# For WASM
248249
async-stream = "0.3.0"

‎lib/vector-api-client/graphql/schema.json

+267
Original file line numberDiff line numberDiff line change
@@ -816,6 +816,95 @@
816816
"name": "ErrorsTotal",
817817
"possibleTypes": null
818818
},
819+
{
820+
"description": "Encoding format for the event",
821+
"enumValues": [
822+
{
823+
"deprecationReason": null,
824+
"description": null,
825+
"isDeprecated": false,
826+
"name": "JSON"
827+
},
828+
{
829+
"deprecationReason": null,
830+
"description": null,
831+
"isDeprecated": false,
832+
"name": "YAML"
833+
}
834+
],
835+
"fields": null,
836+
"inputFields": null,
837+
"interfaces": null,
838+
"kind": "ENUM",
839+
"name": "EventEncodingType",
840+
"possibleTypes": null
841+
},
842+
{
843+
"description": "A notification regarding events observation",
844+
"enumValues": null,
845+
"fields": [
846+
{
847+
"args": [],
848+
"deprecationReason": null,
849+
"description": "Name of the component associated with the notification",
850+
"isDeprecated": false,
851+
"name": "componentName",
852+
"type": {
853+
"kind": "NON_NULL",
854+
"name": null,
855+
"ofType": {
856+
"kind": "SCALAR",
857+
"name": "String",
858+
"ofType": null
859+
}
860+
}
861+
},
862+
{
863+
"args": [],
864+
"deprecationReason": null,
865+
"description": "Event notification type",
866+
"isDeprecated": false,
867+
"name": "notification",
868+
"type": {
869+
"kind": "NON_NULL",
870+
"name": null,
871+
"ofType": {
872+
"kind": "ENUM",
873+
"name": "EventNotificationType",
874+
"ofType": null
875+
}
876+
}
877+
}
878+
],
879+
"inputFields": null,
880+
"interfaces": [],
881+
"kind": "OBJECT",
882+
"name": "EventNotification",
883+
"possibleTypes": null
884+
},
885+
{
886+
"description": "Event notification type",
887+
"enumValues": [
888+
{
889+
"deprecationReason": null,
890+
"description": "A component was found that matched the provided pattern",
891+
"isDeprecated": false,
892+
"name": "MATCHED"
893+
},
894+
{
895+
"deprecationReason": null,
896+
"description": "There isn't currently a component that matches this pattern",
897+
"isDeprecated": false,
898+
"name": "NOT_MATCHED"
899+
}
900+
],
901+
"fields": null,
902+
"inputFields": null,
903+
"interfaces": null,
904+
"kind": "ENUM",
905+
"name": "EventNotificationType",
906+
"possibleTypes": null
907+
},
819908
{
820909
"description": null,
821910
"enumValues": null,
@@ -1627,6 +1716,88 @@
16271716
"name": "LoadAverageMetrics",
16281717
"possibleTypes": null
16291718
},
1719+
{
1720+
"description": "Log event with fields for querying log data",
1721+
"enumValues": null,
1722+
"fields": [
1723+
{
1724+
"args": [],
1725+
"deprecationReason": null,
1726+
"description": "Name of the component associated with the log event",
1727+
"isDeprecated": false,
1728+
"name": "componentName",
1729+
"type": {
1730+
"kind": "NON_NULL",
1731+
"name": null,
1732+
"ofType": {
1733+
"kind": "SCALAR",
1734+
"name": "String",
1735+
"ofType": null
1736+
}
1737+
}
1738+
},
1739+
{
1740+
"args": [],
1741+
"deprecationReason": null,
1742+
"description": "Log message",
1743+
"isDeprecated": false,
1744+
"name": "message",
1745+
"type": {
1746+
"kind": "SCALAR",
1747+
"name": "String",
1748+
"ofType": null
1749+
}
1750+
},
1751+
{
1752+
"args": [],
1753+
"deprecationReason": null,
1754+
"description": "Log timestamp",
1755+
"isDeprecated": false,
1756+
"name": "timestamp",
1757+
"type": {
1758+
"kind": "SCALAR",
1759+
"name": "DateTime",
1760+
"ofType": null
1761+
}
1762+
},
1763+
{
1764+
"args": [
1765+
{
1766+
"defaultValue": null,
1767+
"description": null,
1768+
"name": "encoding",
1769+
"type": {
1770+
"kind": "NON_NULL",
1771+
"name": null,
1772+
"ofType": {
1773+
"kind": "ENUM",
1774+
"name": "EventEncodingType",
1775+
"ofType": null
1776+
}
1777+
}
1778+
}
1779+
],
1780+
"deprecationReason": null,
1781+
"description": "Log event as an encoded string format",
1782+
"isDeprecated": false,
1783+
"name": "string",
1784+
"type": {
1785+
"kind": "NON_NULL",
1786+
"name": null,
1787+
"ofType": {
1788+
"kind": "SCALAR",
1789+
"name": "String",
1790+
"ofType": null
1791+
}
1792+
}
1793+
}
1794+
],
1795+
"inputFields": null,
1796+
"interfaces": [],
1797+
"kind": "OBJECT",
1798+
"name": "Log",
1799+
"possibleTypes": null
1800+
},
16301801
{
16311802
"description": "Host memory metrics",
16321803
"enumValues": null,
@@ -1963,6 +2134,27 @@
19632134
"name": "NetworkMetrics",
19642135
"possibleTypes": null
19652136
},
2137+
{
2138+
"description": "An event or a notification",
2139+
"enumValues": null,
2140+
"fields": null,
2141+
"inputFields": null,
2142+
"interfaces": null,
2143+
"kind": "UNION",
2144+
"name": "OutputEventsPayload",
2145+
"possibleTypes": [
2146+
{
2147+
"kind": "OBJECT",
2148+
"name": "Log",
2149+
"ofType": null
2150+
},
2151+
{
2152+
"kind": "OBJECT",
2153+
"name": "EventNotification",
2154+
"ofType": null
2155+
}
2156+
]
2157+
},
19662158
{
19672159
"description": "Information about pagination in a connection",
19682160
"enumValues": null,
@@ -3502,6 +3694,81 @@
35023694
"description": null,
35033695
"enumValues": null,
35043696
"fields": [
3697+
{
3698+
"args": [
3699+
{
3700+
"defaultValue": null,
3701+
"description": null,
3702+
"name": "componentNames",
3703+
"type": {
3704+
"kind": "NON_NULL",
3705+
"name": null,
3706+
"ofType": {
3707+
"kind": "LIST",
3708+
"name": null,
3709+
"ofType": {
3710+
"kind": "NON_NULL",
3711+
"name": null,
3712+
"ofType": {
3713+
"kind": "SCALAR",
3714+
"name": "String",
3715+
"ofType": null
3716+
}
3717+
}
3718+
}
3719+
}
3720+
},
3721+
{
3722+
"defaultValue": "500",
3723+
"description": null,
3724+
"name": "interval",
3725+
"type": {
3726+
"kind": "NON_NULL",
3727+
"name": null,
3728+
"ofType": {
3729+
"kind": "SCALAR",
3730+
"name": "Int",
3731+
"ofType": null
3732+
}
3733+
}
3734+
},
3735+
{
3736+
"defaultValue": "100",
3737+
"description": null,
3738+
"name": "limit",
3739+
"type": {
3740+
"kind": "NON_NULL",
3741+
"name": null,
3742+
"ofType": {
3743+
"kind": "SCALAR",
3744+
"name": "Int",
3745+
"ofType": null
3746+
}
3747+
}
3748+
}
3749+
],
3750+
"deprecationReason": null,
3751+
"description": "A stream of events emitted from matched component(s)",
3752+
"isDeprecated": false,
3753+
"name": "outputEvents",
3754+
"type": {
3755+
"kind": "NON_NULL",
3756+
"name": null,
3757+
"ofType": {
3758+
"kind": "LIST",
3759+
"name": null,
3760+
"ofType": {
3761+
"kind": "NON_NULL",
3762+
"name": null,
3763+
"ofType": {
3764+
"kind": "UNION",
3765+
"name": "OutputEventsPayload",
3766+
"ofType": null
3767+
}
3768+
}
3769+
}
3770+
}
3771+
},
35053772
{
35063773
"args": [],
35073774
"deprecationReason": null,

‎src/api/mod.rs

+7
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
11
mod handler;
22
mod schema;
33
mod server;
4+
pub mod tap;
45

56
pub use schema::build_schema;
67
pub use server::Server;
8+
9+
use tokio::sync::oneshot;
10+
11+
// Shutdown channel types used by the server and tap.
12+
type ShutdownTx = oneshot::Sender<()>;
13+
type ShutdownRx = oneshot::Receiver<()>;

‎src/api/schema/events/encoding.rs

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
use async_graphql::Enum;
2+
3+
#[derive(Enum, Copy, Clone, PartialEq, Eq)]
4+
/// Encoding format for the event
5+
pub enum EventEncodingType {
6+
Json,
7+
Yaml,
8+
}

0 commit comments

Comments
 (0)