Skip to content

Commit a1410f6

Browse files
a-rodina-rodin
authored
chore(operations): Support x86_64-pc-windows-msvc target (#1205)
* Control which version of leveldb-sys to use with features Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Initial Windows support Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Add metadata_ext.rs to the source tree Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Run `cargo fmt` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix build for `cfg(unix)` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Add `x86_64-pc-windows-msvc` test to CI Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Re-enable syslog test with `cfg(unix)` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Invoke `msiexec` with `cmd.exe` shell Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Move checkout to the beginning Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix Cargo invocation Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Add `cargo` to `PATH` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix fetching of crates.io index Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Use cmd.exe to ensure that correct Perl executable is used Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix Perl path Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Install CMake Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Clean MSVC build Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Implement interruptions handling for Windows Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix CMake installation Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Disable ANSI colors by default on Windows Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Call `msiexec` from `cmd.exe` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Run tests::test_buffering only with `cfg(unix)` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Limit test threads to 1 Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Use `rt.block_on` instead of `block_on` in `test_max_size` Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Clean up Cargo.toml, fix `shiplift` dependency for Unix Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Restore `rdkafka` commit hash in Cargo.lock Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Support `zip` archives Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Build `x86_64-pc-windows-msvc` archive Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Use correct shell for the build step Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Create zip function to call if from the shell script Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Call `./scripts/build-archive.sh` directly Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Fix ZIP compression Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com> * Verify artifacts Signed-off-by: Alexander Rodin <rodin.alexander@gmail.com>
1 parent 5f39c2f commit a1410f6

File tree

15 files changed

+337
-72
lines changed

15 files changed

+337
-72
lines changed

‎.circleci/config.yml

+142-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,10 @@ test-install-script: &test-install-script
2626
# Main document
2727
#
2828

29-
version: 2
29+
version: 2.1
30+
31+
orbs:
32+
win: circleci/windows@1.0.0
3033

3134
jobs:
3235
#
@@ -76,6 +79,54 @@ jobs:
7679
name: Check version
7780
command: make check-version
7881

82+
test-x86_64-pc-windows-msvc:
83+
executor: win/vs2019
84+
steps:
85+
- checkout
86+
- run:
87+
name: Install Rust
88+
shell: bash
89+
command: |
90+
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2019-11-19
91+
# see https://github.com/rust-lang/cargo/issues/2078
92+
printf '[net]\ngit-fetch-with-cli = true\n' > "$HOME/.cargo/config"
93+
- run:
94+
name: Download Perl
95+
shell: bash
96+
command: |
97+
VERSION=5.30.0.1
98+
# we need to verify checksum because strawberryperl.com doesn't support HTTPS
99+
SHA256SUM=459de13a284a4c83213208c9caa1c372c81136b6e863a3f13d42f631048e0b12
100+
curl -sSf http://strawberryperl.com/download/$VERSION/strawberry-perl-$VERSION-64bit.msi > perl-installer.msi
101+
echo "$SHA256SUM perl-installer.msi" | sha256sum --check --status
102+
- run:
103+
name: Install Perl
104+
shell: cmd.exe # msiexec fails when called from bash
105+
command: |
106+
msiexec /quiet /i perl-installer.msi
107+
del perl-installer.msi
108+
- run:
109+
name: Download CMake
110+
shell: bash
111+
command: |
112+
VERSION=3.15.5
113+
curl -sSfL https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-win64-x64.msi > cmake-installer.msi
114+
- run:
115+
name: Install CMake
116+
shell: cmd.exe # msiexec fails when called from bash
117+
command: |
118+
msiexec /quiet /i cmake-installer.msi
119+
del cmake-installer.msi
120+
- run:
121+
name: Build and test
122+
shell: bash
123+
command: |
124+
RUSTFLAGS=-Ctarget-feature=+crt-static
125+
PATH="$HOME/.cargo/bin:/c/Strawberry/perl/bin:/c/Program Files/CMake/bin:$PATH"
126+
FEATURES="leveldb leveldb/leveldb-sys-3 rdkafka rdkafka/cmake_build"
127+
rm rust-toolchain
128+
cargo test --no-default-features --features "$FEATURES" -- --test-threads=1
129+
79130
test-stable:
80131
resource_class: xlarge
81132
docker:
@@ -155,6 +206,68 @@ jobs:
155206
paths:
156207
- "*-x86_64-apple-darwin.tar.gz"
157208

209+
build-x86_64-pc-windows-msvc-archive:
210+
executor: win/vs2019
211+
steps:
212+
- checkout
213+
- run:
214+
name: Install Rust
215+
shell: bash
216+
command: |
217+
curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly-2019-11-19
218+
# see https://github.com/rust-lang/cargo/issues/2078
219+
printf '[net]\ngit-fetch-with-cli = true\n' > "$HOME/.cargo/config"
220+
- run:
221+
name: Download Perl
222+
shell: bash
223+
command: |
224+
VERSION=5.30.0.1
225+
# we need to verify checksum because strawberryperl.com doesn't support HTTPS
226+
SHA256SUM=459de13a284a4c83213208c9caa1c372c81136b6e863a3f13d42f631048e0b12
227+
curl -sSf http://strawberryperl.com/download/$VERSION/strawberry-perl-$VERSION-64bit.msi > perl-installer.msi
228+
echo "$SHA256SUM perl-installer.msi" | sha256sum --check --status
229+
- run:
230+
name: Install Perl
231+
shell: cmd.exe # msiexec fails when called from bash
232+
command: |
233+
msiexec /quiet /i perl-installer.msi
234+
del perl-installer.msi
235+
- run:
236+
name: Download CMake
237+
shell: bash
238+
command: |
239+
VERSION=3.15.5
240+
curl -sSfL https://github.com/Kitware/CMake/releases/download/v$VERSION/cmake-$VERSION-win64-x64.msi > cmake-installer.msi
241+
- run:
242+
name: Install CMake
243+
shell: cmd.exe # msiexec fails when called from bash
244+
command: |
245+
msiexec /quiet /i cmake-installer.msi
246+
del cmake-installer.msi
247+
- run:
248+
name: Build archive
249+
shell: bash
250+
command: |
251+
export PATH="$HOME/.cargo/bin:/c/Strawberry/perl/bin:/c/Program Files/CMake/bin:$PATH"
252+
function zip() {
253+
powershell Compress-Archive $3 $2
254+
}
255+
export -f zip
256+
257+
export RUSTFLAGS=-Ctarget-feature=+crt-static
258+
export FEATURES="leveldb leveldb/leveldb-sys-3 rdkafka rdkafka/cmake_build"
259+
export ARCHIVE_TYPE="zip"
260+
export STRIP="false"
261+
export RUST_LTO=""
262+
export TARGET="x86_64-pc-windows-msvc"
263+
264+
./scripts/build-archive.sh
265+
ls -lha target/artifacts
266+
- persist_to_workspace:
267+
root: target/artifacts
268+
paths:
269+
- "*-x86_64-pc-windows-msvc.zip"
270+
158271
build-x86_64-unknown-linux-gnu-archive-and-deb-package:
159272
docker:
160273
- image: timberiodev/vector-builder-x86_64-unknown-linux-gnu:latest
@@ -362,6 +475,17 @@ jobs:
362475
name: Verify installation
363476
command: vector --version
364477

478+
verify-zip-artifact-on-windows:
479+
executor: win/vs2019
480+
steps:
481+
- *restore-artifacts-from-workspace
482+
- run:
483+
name: Verify package
484+
shell: bash
485+
command: |
486+
powershell Expand-Archive target/artifacts/vector-x86_64-pc-windows-msvc.zip .
487+
./vector-x86_64-pc-windows-msvc/bin/vector.exe --version
488+
365489
verify-docker:
366490
docker:
367491
- image: timberiodev/vector-releaser:latest
@@ -629,12 +753,14 @@ require-tests-checks-and-verifications: &require-tests-checks-and-verifications
629753
- build-aarch64-unknown-linux-musl-archive
630754
- build-armv7-unknown-linux-musleabihf-archive
631755
- build-x86_64-apple-darwin-archive
756+
- build-x86_64-pc-windows-msvc-archive
632757
- verify-deb-artifact-on-deb-8
633758
- verify-deb-artifact-on-deb-9
634759
- verify-deb-artifact-on-deb-10
635760
- verify-deb-artifact-on-ubuntu-16-04
636761
- verify-deb-artifact-on-ubuntu-18-04
637762
- verify-deb-artifact-on-ubuntu-19-04
763+
- verify-zip-artifact-on-windows
638764
- verify-docker
639765
- verify-rpm-artifact-on-amazon-linux-1
640766
- verify-rpm-artifact-on-amazon-linux-2
@@ -657,6 +783,7 @@ workflows:
657783
- check-version:
658784
<<: *master-filters
659785
- test-stable
786+
- test-x86_64-pc-windows-msvc
660787

661788
# Build & verify
662789

@@ -665,6 +792,7 @@ workflows:
665792
- build-aarch64-unknown-linux-musl-archive
666793
- build-armv7-unknown-linux-musleabihf-archive
667794
- build-x86_64-apple-darwin-archive
795+
- build-x86_64-pc-windows-msvc-archive
668796
- package-rpm:
669797
requires:
670798
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
@@ -689,6 +817,9 @@ workflows:
689817
- verify-deb-artifact-on-ubuntu-19-04:
690818
requires:
691819
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
820+
- verify-zip-artifact-on-windows:
821+
requires:
822+
- build-x86_64-pc-windows-msvc-archive
692823
- verify-docker:
693824
requires:
694825
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
@@ -729,6 +860,8 @@ workflows:
729860
<<: *release-filters
730861
- build-x86_64-apple-darwin-archive:
731862
<<: *release-filters
863+
- build-x86_64-pc-windows-msvc-archive:
864+
<<: *release-filters
732865
- package-rpm:
733866
<<: *release-filters
734867
requires:
@@ -760,6 +893,10 @@ workflows:
760893
<<: *release-filters
761894
requires:
762895
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
896+
- verify-zip-artifact-on-windows:
897+
<<: *release-filters
898+
requires:
899+
- build-x86_64-pc-windows-msvc-archive
763900
- verify-docker:
764901
<<: *release-filters
765902
requires:
@@ -819,6 +956,7 @@ workflows:
819956
- build-aarch64-unknown-linux-musl-archive
820957
- build-armv7-unknown-linux-musleabihf-archive
821958
- build-x86_64-apple-darwin-archive
959+
- build-x86_64-pc-windows-msvc-archive
822960
- package-rpm:
823961
requires:
824962
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
@@ -843,6 +981,9 @@ workflows:
843981
- verify-deb-artifact-on-ubuntu-19-04:
844982
requires:
845983
- build-x86_64-unknown-linux-gnu-archive-and-deb-package
984+
- verify-zip-artifact-on-windows:
985+
requires:
986+
- build-x86_64-pc-windows-msvc-archive
846987
- verify-docker:
847988
requires:
848989
- build-x86_64-unknown-linux-gnu-archive-and-deb-package

‎Cargo.lock

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+5-3
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,6 @@ prost-derive = "0.5"
8383
prost-types = "0.5"
8484

8585
# External libs
86-
atty = "0.2"
8786
derivative = "1.0"
8887
chrono = "0.4.6"
8988
rand = "0.5.5"
@@ -124,13 +123,16 @@ exitcode = "1.1.2"
124123
snafu = "0.4.3"
125124
url = "1.7"
126125
base64 = "0.10.1"
127-
shiplift = { git = "https://github.com/LucioFranco/shiplift", branch = "timber" }
126+
shiplift = { git = "https://github.com/LucioFranco/shiplift", branch = "timber", default-features = false, features = ["tls"] }
128127
owning_ref = "0.4.0"
129128
listenfd = "0.3.3"
130129
inventory = "0.1"
131130
maxminddb = "0.13.0"
132131
strip-ansi-escapes = "0.1.0"
133132

133+
[target.'cfg(unix)'.dependencies]
134+
atty = "0.2"
135+
134136
[build-dependencies]
135137
prost-build = "0.4.0"
136138
built = "0.3"
@@ -150,7 +152,7 @@ tokio01-test = "0.1.1"
150152
tower-test = "0.1"
151153

152154
[features]
153-
default = ["rdkafka", "leveldb", "leveldb/leveldb-sys-2", "jemallocator"]
155+
default = ["rdkafka", "leveldb", "leveldb/leveldb-sys-2", "shiplift/unix-socket", "jemallocator"]
154156
docker = [
155157
"cloudwatch-logs-integration-tests",
156158
"cloudwatch-metrics-integration-tests",

‎lib/file-source/src/file_server.rs

+4-3
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@ use indexmap::IndexMap;
66
use std::collections::{HashMap, HashSet};
77
use std::fs;
88
use std::io::{self, Read, Seek, Write};
9-
use std::os::unix::fs::MetadataExt;
109
use std::path::{Path, PathBuf};
1110
use std::sync::mpsc::RecvTimeoutError;
1211
use std::time;
1312
use tracing::field;
1413

14+
use crate::metadata_ext::PortableMetadataExt;
15+
1516
/// `FileServer` is a Source which cooperatively schedules reads over files,
1617
/// converting the lines of said files into `LogLine` structures. As
1718
/// `FileServer` is intended to be useful across multiple operating systems with
@@ -392,8 +393,8 @@ impl Fingerprinter {
392393
match *self {
393394
Fingerprinter::DevInode => {
394395
let metadata = fs::metadata(path)?;
395-
let dev = metadata.dev();
396-
let ino = metadata.ino();
396+
let dev = metadata.portable_dev();
397+
let ino = metadata.portable_ino();
397398
buffer.clear();
398399
buffer.write_all(&dev.to_be_bytes())?;
399400
buffer.write_all(&ino.to_be_bytes())?;

‎lib/file-source/src/file_watcher.rs

+7-6
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
use crate::FilePosition;
22
use std::fs;
33
use std::io::{self, BufRead, Seek};
4-
use std::os::unix::fs::MetadataExt;
54
use std::path::PathBuf;
65
use std::time;
76

7+
use crate::metadata_ext::PortableMetadataExt;
8+
89
/// The `FileWatcher` struct defines the polling based state machine which reads
910
/// from a file path, transparently updating the underlying file descriptor when
1011
/// the file has been rolled over, as is common for logs.
@@ -56,20 +57,20 @@ impl FileWatcher {
5657
findable: true,
5758
reader: rdr,
5859
file_position,
59-
devno: metadata.dev(),
60-
inode: metadata.ino(),
60+
devno: metadata.portable_dev(),
61+
inode: metadata.portable_ino(),
6162
is_dead: false,
6263
})
6364
}
6465

6566
pub fn update_path(&mut self, path: PathBuf) -> io::Result<()> {
6667
let metadata = fs::metadata(&path)?;
67-
if (metadata.dev(), metadata.ino()) != (self.devno, self.inode) {
68+
if (metadata.portable_dev(), metadata.portable_ino()) != (self.devno, self.inode) {
6869
let mut new_reader = io::BufReader::new(fs::File::open(&path)?);
6970
new_reader.seek(io::SeekFrom::Start(self.file_position))?;
7071
self.reader = new_reader;
71-
self.devno = metadata.dev();
72-
self.inode = metadata.ino();
72+
self.devno = metadata.portable_dev();
73+
self.inode = metadata.portable_ino();
7374
}
7475
self.path = path;
7576
Ok(())

‎lib/file-source/src/lib.rs

+2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1+
#![cfg_attr(windows, feature(windows_by_handle))]
12
#[macro_use]
23
extern crate scan_fmt;
34
#[macro_use]
45
extern crate tracing;
56

67
mod file_server;
78
mod file_watcher;
9+
mod metadata_ext;
810

911
pub use self::file_server::{FileServer, Fingerprinter};
1012

‎lib/file-source/src/metadata_ext.rs

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
use std::fs::Metadata;
2+
#[cfg(unix)]
3+
use std::os::unix::fs::MetadataExt;
4+
#[cfg(windows)]
5+
use std::os::windows::fs::MetadataExt;
6+
7+
pub trait PortableMetadataExt {
8+
fn portable_dev(&self) -> u64;
9+
fn portable_ino(&self) -> u64;
10+
}
11+
12+
#[cfg(unix)]
13+
impl PortableMetadataExt for Metadata {
14+
fn portable_dev(&self) -> u64 {
15+
self.dev()
16+
}
17+
fn portable_ino(&self) -> u64 {
18+
self.ino()
19+
}
20+
}
21+
22+
#[cfg(windows)]
23+
impl PortableMetadataExt for Metadata {
24+
fn portable_dev(&self) -> u64 {
25+
self.volume_serial_number().unwrap_or(0u32) as u64
26+
}
27+
fn portable_ino(&self) -> u64 {
28+
self.file_index().unwrap_or(0u64)
29+
}
30+
}

0 commit comments

Comments
 (0)