uplink-sys(test): Run integration tests in storj-up environment

Previously the `uplink-sys` integration tests had to use a Storj network
running somewhere or a external local environment that people had to set
it up manually.

The `uplink` crate was already using `storj-up` for its integration
tests.

This commit generalize the `storj-up` configuration used in the `uplink`
crate to make available to both crates and adapt the `uplink-sys`
Makefile, CI (Github workflow), and integration tests to use the
`storj-up` environment.

The changes make in the `uplink-sys` integration tests are only for
making easier to adapt it to `storj-up` environment despite, now, it
uses another function to get the access grant, the integration test was
not exactly testing the previous function, so it has not changed the end
goal of it.

There is a minimal change in the CI (Github workflow) of the `uplink`
crate that is out of the scope of this commit, but the changes are
minimal and shouldn't confuse the reviewers. The changes are for adding
a new step to the `check-implementation` job to simulate publishing the
crate in order to verify that any merged PR shouldn't break its version
publishing.

Author: ifraixedes

.github/workflows/uplink-sys.yml

@@ -27,23 +27,14 @@ jobs:
       run: git submodule update --init
     - name: Lint
       run: make lint
-    # Clippy
-    #- name: clippy
-      # workaround for clippy bug where it emits warnings instead of errors if it's not a clean build (or in this case we just fudge the timestamps of .rs files)
-      #run: find . | grep "\.rs$" | xargs touch ; cargo clippy --workspace -- -D clippy::all
     - name: Build
       run: make build
     - name: Check lib binaries for docs.rs
       run: | # The way how to check if the libs are have to date with only comparing libuplink.pc isn't ideal but it's what we have found that works.
         make update-libs-docs-rs
         test -z $(git ls-files -d -m -o .docs-rs/libuplink.pc) || (printf "the uplink-c library binaries for docs.rs aren't up to date. Update them with `make update-libs-docs-rs`\n" && exit 1)
         git clean -fd && git checkout .
-    - name: Create secrets for tests
-      run: printf "${{ secrets.STORJ_TEST_SAT_ADDRESS }}\n${{ secrets.STORJ_TEST_API_KEY }}\n${{ secrets.STORJ_TEST_PASSPHRASE }}" > test_secrets.txt
     - name: Test
       run: make test
-    - name: Delete secrets for tests
-      if: always()
-      run: rm -f test_secrets.txt
-    - name: Crate Publish Test
+    - name: Publish crate simulation
       run: make publish-test

.github/workflows/uplink.yml

@@ -32,6 +32,8 @@ jobs:
       - name: Clean up
         if: always()
         run: make clean
+      - name: Publish crate simulation
+        run: make publish-test
   test-implementation:
     runs-on: ubuntu-latest
     defaults:

.gitignore

@@ -1,2 +1,3 @@
 /target
 Cargo.lock
+.tmp/

Makefile

@@ -20,6 +20,38 @@ test:
 	$(MAKE) -C uplink test
 
 .PHONY: clean
-clean:
+clean: integration-tests-env-down
+	rm -rf .tmp
 	$(MAKE) -C uplink-sys clean
 	$(MAKE) -C uplink clean
+
+
+.PHONY: integration-tests-env-up
+integration-tests-env-up:
+	docker compose up -d
+	$(MAKE) .tmp/env
+
+.PHONY: integration-tests-env-down
+integration-tests-env-down:
+	docker compose down
+	rm -rf .tmp
+
+.tmp/env: .tmp/up/storj-up
+	@.tmp/up/storj-up credentials -e | grep -Ei "^export .+" > .tmp/env
+	# TODO: This is a hack to get the AWS_* and STORJ_GATEWAY variables without
+	# overriding the access grants because those variable are only available using
+	# the storj-up from inside of the container, however, we cannot use the access
+	# grants because they use the docker compose service name in the URL rather
+	# than localhost and then it doesn't resolve.
+	# See: https://github.com/storj/up/issues/45#issuecomment-1288808260
+	@docker compose exec -T satellite-api storj-up credentials --s3 -e \
+		-a http://authservice:8888 -s satellite-api \
+		| grep -E 'AWS|STORJ_GATEWAY' >> .tmp/env
+
+.tmp/up/storj-up: .tmp/up
+	cd .tmp/up; go build -o storj-up
+
+.tmp/up:
+	mkdir -p .tmp
+	cd .tmp; git clone https://github.com/storj/up.git
+	cd .tmp/up; git checkout v1.1.0

README.md

@@ -9,11 +9,20 @@ Storj Uplink Rust bindings for the Rust programming language.
 
 ## Repository layout
 
-Following the conventions used for creating Rust bindings through [bindgen][bindgen], this repository contains two crates:
+Following the conventions used for creating Rust bindings through [bindgen][bindgen], this
+repository contains two crates:
 
 * The [`uplink-sys`](uplink-sys) which is the unsafe Rust bindings auto-generated by [bindgen][bindgen].
 * The [`uplink`](uplink) which is the safe and idiomatic Rust binding build on top of the `uplink-sys`.
 
 [bindgen]: https://github.com/rust-lang/rust-bindgen/
 
-Each crate matches a root's child directory with the same name and each directory has its own README which provides more detailed information and its current status.
+Each crate matches a root's child directory with the same name and each directory has its own README
+which provides more detailed information and its current status.
+
+The root directory of this repository, apart of Cargo files for defining a workspace and this
+README, it contains a _Makefile_ with targets to execute operations in both crates and operations that
+are shared between both crates.
+
+The _docker-compose.yaml_ file is for executing all the external services that the integration tests
+requires.

uplink/tests/docker-compose.yaml docker-compose.yaml

@@ -1,3 +1,2 @@
 /target
 Cargo.lock
-test_secrets.txt

uplink-sys/.gitignore

@@ -6,6 +6,7 @@ LOCAL_ABS_CRATE_PATH := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 # Makefile special variables #
 
 .DEFAULT_GOAL := build
+SHELL := /bin/bash
 
 # Targets #
 
@@ -20,7 +21,8 @@ lint:
 
 .PHONY: test
 test: $(UPLINK_C)/.git
-	cargo test
+	$(MAKE) -C .. integration-tests-env-up
+	source ../.tmp/env; cargo test
 
 .PHONY: publish-test
 publish-test: clean $(UPLINK_C)/.git

uplink-sys/Makefile

@@ -1,28 +1,16 @@
+use std::env;
 use std::ffi::{CStr, CString};
 use std::fs;
 
 #[test]
 fn list_buckets() {
-    // Get secrets for accessing project
-    let secrets = fs::read_to_string("test_secrets.txt").unwrap();
-    let secrets = secrets.lines().collect::<Vec<&str>>();
-    assert_eq!(
-        secrets.len(),
-        3,
-        "test_secrets.txt file has to contain 3 lines, satellite address, API key, and passphrase"
-    );
-
-    // Access parameters
-    let satellite_address = CString::new(secrets[0]).expect("CString::new failed");
-    let api_key = CString::new(secrets[1]).expect("CString::new failed");
-    let passphrase = CString::new(secrets[2]).expect("CString::new failed");
+    let access_grant = env::var("STORJ_ACCESS").expect("STORJ_ACCESS env var isn't defined");
+    let access_grant = CString::new(access_grant).expect("CString::new failed");
 
     unsafe {
-        // Request access
-        let access_result = uplink_sys::uplink_request_access_with_passphrase(
-            satellite_address.as_ptr() as *mut uplink_sys::uplink_const_char,
-            api_key.as_ptr() as *mut uplink_sys::uplink_const_char,
-            passphrase.as_ptr() as *mut uplink_sys::uplink_const_char,
+        // Parse access grant
+        let access_result = uplink_sys::uplink_parse_access(
+            access_grant.as_ptr() as *mut uplink_sys::uplink_const_char
         );
 
         assert_eq!(access_result.error, std::ptr::null_mut()); // verify no error

uplink-sys/tests/list_buckets.rs

@@ -1,3 +1,2 @@
 /target
 Cargo.lock
-tests/.tmp/

uplink/.gitignore

@@ -23,48 +23,18 @@ test-doc:
 	cargo test --doc
 
 .PHONY: test-integration
-test-integration: integration-tests-env-up tests/.tmp/env
+test-integration:
+	$(MAKE) -C .. integration-tests-env-up
 	# We don't use `cargo test --test` because it requires to pass a specific test file name and
 	# it doesn't accept a list of files. For being able to only run the integration tests we prefix
 	# all the integration tests functions with `integration_` and then filter them with the usual
 	# `cargo test` command.
-	source tests/.tmp/env; cargo test integration
+	source ../.tmp/env; cargo test integration
 
 .PHONY: test-unit
 test-unit:
 	cargo test --lib
 
-tests/.tmp/env: tests/.tmp/up/storj-up
-	$(MAKE) integration-tests-credentials | grep -Ei "^export .+" > tests/.tmp/env
-	# TODO(github.com/storj-thirdparty/uplink-rust/issues/48): This is a hack to get the AWS_* and
-	# STORJ_GATEWAY variables without overriding the access grants because those variable are only
-	# available using the storj-up from inside of the container, however, we cannot use the access
-	# grants because they use the docker compose service name in the URL rather than localhost and
-	# then it doesn't resolve.
-	@cd tests; docker compose exec -T satellite-api storj-up credentials --s3 -e \
-		-a http://authservice:8888 -s satellite-api \
-		| grep -E 'AWS|STORJ_GATEWAY' >> .tmp/env
-
-.PHONY: integration-tests-credentials
-integration-tests-credentials: tests/.tmp/up/storj-up
-	@cd tests; .tmp/up/storj-up credentials -e
-
-.PHONY: integration-tests-env-up
-integration-tests-env-up:
-	cd tests; docker compose up -d
-
-.PHONY: .integration-tests-env-down
-.integration-tests-env-down:
-	cd tests; docker compose down
-
-tests/.tmp/up/storj-up: tests/.tmp/up
-	cd tests/.tmp/up; go build -o storj-up
-
-tests/.tmp/up:
-	mkdir -p tests/.tmp
-	cd tests/.tmp; git clone https://github.com/storj/up.git
-	cd tests/.tmp/up; git checkout v1.1.0
-
 ## Publish crate ##
 .PHONY: publish-test
 publish-test:
@@ -76,6 +46,5 @@ _publish-crate:
 
 ## Clean ##
 .PHONY: clean
-clean: .integration-tests-env-down
-	rm -rf tests/.tmp
+clean:
 	cargo clean