Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up #11

Merged
merged 16 commits into from
Nov 2, 2019
Merged

Clean up #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
c643f53
Remove mistaken condition from mergify config
alecmocatta Oct 17, 2019
a5969ff
Merge pull request #4 from alecmocatta/mergify-fix
alecmocatta Oct 17, 2019
6aea044
Merge remote-tracking branch 'template/master' into clean
alecmocatta Oct 19, 2019
69d62b6
clippy clean
alecmocatta Oct 19, 2019
a3dcd50
clean pool apis
alecmocatta Oct 20, 2019
74b6308
Update serde_traitobject requirement from 0.1.6 to 0.2.2
dependabot-preview[bot] Oct 21, 2019
8fd1663
Add .editorconfig for tab_width = 4
alecmocatta Oct 21, 2019
bd2e1ea
Merge pull request #5 from alecmocatta/alecmocatta-patch-1
mergify[bot] Oct 21, 2019
1ee8d24
clean upo downcast and serde apis
alecmocatta Oct 23, 2019
ed07254
Update Cargo.toml
alecmocatta Oct 25, 2019
dbf350b
generate parquet's thrift bindings from definition
alecmocatta Nov 1, 2019
39d2e20
Merge remote-tracking branch 'origin/dependabot/cargo/serde_traitobje…
alecmocatta Nov 1, 2019
3458292
Merge remote-tracking branch 'template/master' into clean
alecmocatta Nov 1, 2019
1eadbd0
Map -> HashMap
alecmocatta Nov 1, 2019
c4792a4
minor fixes
alecmocatta Nov 2, 2019
859912f
v0.1.4
alecmocatta Nov 2, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .editorconfig
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[*]
tab_width = 4

[.*]
tab_width = 4
8 changes: 8 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,11 @@
path = amadeus-testing
url = https://github.com/alecmocatta/amadeus-testing.git
branch = master
[submodule "amadeus-parquet/thrift"]
path = amadeus-parquet/thrift
url = https://github.com/apache/thrift.git
branch = master
[submodule "amadeus-parquet/parquet-format"]
path = amadeus-parquet/parquet-format
url = https://github.com/apache/parquet-format.git
branch = master
2 changes: 0 additions & 2 deletions .mergify.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ pull_request_rules:
- "#approved-reviews-by>=1"
- "#review-requested=0"
- "#changes-requested-reviews-by=0"
- "#commented-reviews-by=0"
actions:
merge:
method: merge
Expand All @@ -21,7 +20,6 @@ pull_request_rules:
- author=alecmocatta # https://github.com/Mergifyio/mergify-engine/issues/451
- "#review-requested=0"
- "#changes-requested-reviews-by=0"
- "#commented-reviews-by=0"
actions:
merge:
method: merge
Expand Down
25 changes: 11 additions & 14 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

[package]
name = "amadeus"
version = "0.1.3"
version = "0.1.4"
license = "Apache-2.0"
authors = ["Alec Mocatta <alec@mocatta.net>"]
categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
Expand All @@ -14,7 +14,7 @@ parquet postgres aws s3 cloudfront elb json csv logs hadoop hdfs arrow common cr
"""
repository = "https://github.com/alecmocatta/amadeus"
homepage = "https://github.com/alecmocatta/amadeus"
documentation = "https://docs.rs/amadeus/0.1.3"
documentation = "https://docs.rs/amadeus/0.1.4"
readme = "README.md"
edition = "2018"

Expand All @@ -36,23 +36,20 @@ doc = ["amadeus-aws/doc", "amadeus-commoncrawl/doc", "amadeus-parquet/doc", "ama
features = ["doc", "constellation", "aws", "commoncrawl", "postgres", "csv", "json"]

[dependencies]
amadeus-core = { version = "=0.1.3", path = "amadeus-core" }
amadeus-derive = { version = "=0.1.3", path = "amadeus-derive" }
amadeus-types = { version = "=0.1.3", path = "amadeus-types" }
amadeus-aws = { version = "=0.1.3", path = "amadeus-aws", optional = true }
amadeus-commoncrawl = { version = "=0.1.3", path = "amadeus-commoncrawl", optional = true }
amadeus-parquet = { version = "=0.1.3", path = "amadeus-parquet", optional = true }
amadeus-postgres = { version = "=0.1.3", path = "amadeus-postgres", optional = true }
amadeus-serde = { version = "=0.1.3", path = "amadeus-serde", optional = true }
amadeus-core = { version = "=0.1.4", path = "amadeus-core" }
amadeus-derive = { version = "=0.1.4", path = "amadeus-derive" }
amadeus-types = { version = "=0.1.4", path = "amadeus-types" }
amadeus-aws = { version = "=0.1.4", path = "amadeus-aws", optional = true }
amadeus-commoncrawl = { version = "=0.1.4", path = "amadeus-commoncrawl", optional = true }
amadeus-parquet = { version = "=0.1.4", path = "amadeus-parquet", optional = true }
amadeus-postgres = { version = "=0.1.4", path = "amadeus-postgres", optional = true }
amadeus-serde = { version = "=0.1.4", path = "amadeus-serde", optional = true }
constellation-rs = { version = "0.1", default-features = false, optional = true }
futures-preview = "=0.3.0-alpha.19"
pin-utils = "0.1.0-alpha.4"
serde = { version = "1.0", features = ["derive"] }
serde_closure = "0.2"
serde_traitobject = "0.1.6"

# pin; broken for some reason
bincode = "=1.0.1"
serde_traitobject = "0.2"

[dev-dependencies]
either = { version = "1.5", features = ["serde"] }
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
</p>

<p align="center">
<a href="https://docs.rs/amadeus/0.1.3">Docs</a>
<a href="https://docs.rs/amadeus/0.1.4">Docs</a>
</p>


Expand Down
8 changes: 4 additions & 4 deletions amadeus-aws/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "amadeus-aws"
version = "0.1.3"
version = "0.1.4"
license = "Apache-2.0"
authors = ["Alec Mocatta <alec@mocatta.net>"]
categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
Expand All @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
"""
repository = "https://github.com/alecmocatta/amadeus"
homepage = "https://github.com/alecmocatta/amadeus"
documentation = "https://docs.rs/amadeus/0.1.3"
documentation = "https://docs.rs/amadeus/0.1.4"
readme = "README.md"
edition = "2018"

Expand All @@ -22,8 +22,8 @@ maintenance = { status = "actively-developed" }
doc = []

[dependencies]
amadeus-core = { version = "=0.1.3", path = "../amadeus-core" }
amadeus-types = { version = "=0.1.3", path = "../amadeus-types" }
amadeus-core = { version = "=0.1.4", path = "../amadeus-core" }
amadeus-types = { version = "=0.1.4", path = "../amadeus-types" }
chrono = { version = "0.4", default-features = false }
flate2 = "1.0"
futures-01 = { package = "futures", version = "0.1" }
Expand Down
11 changes: 8 additions & 3 deletions amadeus-aws/src/cloudfront.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ impl Source for Cloudfront {
.into_dist_iter()
.flat_map(FnMut!(move |key: String| {
let client = S3Client::new(region.clone());
let mut errors = 0;
ResultExpand(
loop {
match self::block_on_01(self::retry(|| {
Expand All @@ -68,11 +69,15 @@ impl Source for Cloudfront {
..GetObjectRequest::default()
})
})) {
Err(RusotoError::HttpDispatch(_)) => continue,
Err(RusotoError::HttpDispatch(_)) if errors < 10 => {
errors += 1;
continue;
}
Err(RusotoError::Unknown(response))
if response.status.is_server_error() =>
if response.status.is_server_error() && errors < 10 =>
{
continue
errors += 1;
continue;
}
res => break res,
}
Expand Down
13 changes: 10 additions & 3 deletions amadeus-aws/src/file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ impl Page for S3Page {
Box::pin(async move {
let len: u64 = buf.len().try_into().unwrap();
let mut cursor = io::Cursor::new(buf);
let mut errors = 0;
while len - cursor.position() > 0 {
let (start, end) = (offset + cursor.position(), offset + len - 1);
let res = Compat01As03::new(self.client.get_object(GetObjectRequest {
Expand All @@ -216,9 +217,15 @@ impl Page for S3Page {
}));
let res = res.await;
match res {
Err(RusotoError::HttpDispatch(_)) => continue,
Err(RusotoError::Unknown(response)) if response.status.is_server_error() => {
continue
Err(RusotoError::HttpDispatch(_)) if errors < 10 => {
errors += 1;
continue;
}
Err(RusotoError::Unknown(response))
if response.status.is_server_error() && errors < 10 =>
{
errors += 1;
continue;
}
_ => (),
}
Expand Down
2 changes: 1 addition & 1 deletion amadeus-aws/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.1.3")]
#![doc(html_root_url = "https://docs.rs/amadeus-aws/0.1.4")]
#![feature(type_alias_impl_trait)]

mod cloudfront;
Expand Down
8 changes: 4 additions & 4 deletions amadeus-commoncrawl/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "amadeus-commoncrawl"
version = "0.1.3"
version = "0.1.4"
license = "MIT OR Apache-2.0"
authors = ["Stephen Becker IV <github@deathbyescalator.com>", "Alec Mocatta <alec@mocatta.net>"]
categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
Expand All @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
"""
repository = "https://github.com/alecmocatta/amadeus"
homepage = "https://github.com/alecmocatta/amadeus"
documentation = "https://docs.rs/amadeus/0.1.3"
documentation = "https://docs.rs/amadeus/0.1.4"
readme = "README.md"
edition = "2018"

Expand All @@ -22,8 +22,8 @@ maintenance = { status = "actively-developed" }
doc = []

[dependencies]
amadeus-core = { version = "=0.1.3", path = "../amadeus-core" }
amadeus-types = { version = "=0.1.3", path = "../amadeus-types" }
amadeus-core = { version = "=0.1.4", path = "../amadeus-core" }
amadeus-types = { version = "=0.1.4", path = "../amadeus-types" }
flate2 = "1.0"
reqwest = "0.9"
reqwest_resume = "0.2"
Expand Down
2 changes: 1 addition & 1 deletion amadeus-commoncrawl/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.1.3")]
#![doc(html_root_url = "https://docs.rs/amadeus-commoncrawl/0.1.4")]
#![feature(type_alias_impl_trait)]

mod commoncrawl;
Expand Down
4 changes: 2 additions & 2 deletions amadeus-core/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "amadeus-core"
version = "0.1.3"
version = "0.1.4"
license = "Apache-2.0"
authors = ["Alec Mocatta <alec@mocatta.net>"]
categories = ["concurrency", "science", "database", "parser-implementations", "text-processing"]
Expand All @@ -10,7 +10,7 @@ Harmonious distributed data analysis in Rust.
"""
repository = "https://github.com/alecmocatta/amadeus"
homepage = "https://github.com/alecmocatta/amadeus"
documentation = "https://docs.rs/amadeus/0.1.3"
documentation = "https://docs.rs/amadeus/0.1.4"
readme = "README.md"
edition = "2018"

Expand Down
Loading