Skip to content

Commit 4c68df5

Browse files
snackksnackk
authored
ID-1570 - Fix dice-where and update dce-id to use the latest version (#113)
* ID-1570 - Alternative way; * ID-1570 - working version; * ID-1570 - unnecessary code; * ID-1570 - unnecessary imports;
1 parent 6703b9d commit 4c68df5

File tree

6 files changed

+182
-59
lines changed

6 files changed

+182
-59
lines changed

‎dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/FileAcceptor.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88

99
public interface FileAcceptor<T> {
1010

11-
StreamConsumer<T> getStreamConsumer(MD5Checksum originalFileMd5, Instant originalFileTimestamp,
12-
boolean noMd5Check);
11+
StreamConsumer<T> getStreamConsumer(
12+
MD5Checksum originalFileMd5, Instant originalFileTimestamp, boolean noMd5Check);
1313

1414
boolean destinationExists();
1515

‎dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptor.java

+2-4
Original file line numberDiff line numberDiff line change
@@ -73,13 +73,11 @@ public Optional<MD5Checksum> existingFileMd5() {
7373
BufferedInputStream bis = new BufferedInputStream(is);
7474
StreamWithMD5Decorator md5Is = StreamWithMD5Decorator.of(bis)) {
7575
byte[] buffer = new byte[BUFFER];
76-
while ((md5Is.read(buffer)) != -1) {
77-
}
76+
while ((md5Is.read(buffer)) != -1) {}
7877
return Optional.of(md5Is.md5());
7978
} catch (IOException | NoSuchAlgorithmException e) {
8079
throw new RuntimeException(
81-
"Could not obtain md5 of the file existing at the target: " + destination,
82-
e);
80+
"Could not obtain md5 of the file existing at the target: " + destination, e);
8381
}
8482
}
8583
return Optional.empty();

‎dice-where-downloader-lib/src/main/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5Decorator.java

+15-4
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,15 @@
55
import java.security.DigestInputStream;
66
import java.security.MessageDigest;
77
import java.security.NoSuchAlgorithmException;
8+
import java.util.Optional;
89
import javax.xml.bind.annotation.adapters.HexBinaryAdapter;
910
import technology.dice.dicewhere.downloader.md5.MD5Checksum;
1011

1112
public class StreamWithMD5Decorator extends InputStream {
1213

1314
private final MessageDigest md5;
1415
DigestInputStream inputStream;
16+
private Optional<MD5Checksum> checksum = Optional.empty();
1517

1618
private StreamWithMD5Decorator(DigestInputStream inputStream, MessageDigest md5) {
1719
this.inputStream = inputStream;
@@ -25,17 +27,26 @@ public static StreamWithMD5Decorator of(InputStream inputStream) throws NoSuchAl
2527
}
2628

2729
public MD5Checksum md5() {
28-
String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
29-
return MD5Checksum.of(hex);
30+
return checksum.orElseGet(
31+
() -> {
32+
String hex = (new HexBinaryAdapter()).marshal(this.md5.digest());
33+
checksum = Optional.of(MD5Checksum.of(hex));
34+
return checksum.get();
35+
});
3036
}
3137

3238
@Override
3339
public int read() throws IOException {
34-
return this.inputStream.read();
40+
return inputStream.read();
41+
}
42+
43+
@Override
44+
public int read(byte[] b, int off, int len) throws IOException {
45+
return inputStream.read(b, off, len);
3546
}
3647

3748
@Override
3849
public void close() throws IOException {
39-
this.inputStream.close();
50+
inputStream.close();
4051
}
4152
}

‎dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/destination/local/LocalFileAcceptorTest.java

+82-49
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@
3333
public class LocalFileAcceptorTest extends TestCase {
3434

3535
private static final int TEST_FILE_SIZE = 1024 * 1024;
36-
@ClassRule
37-
static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
36+
@ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
3837

3938
@BeforeClass
4039
public static void beforeClass() {
@@ -45,16 +44,22 @@ public static void beforeClass() {
4544
public void corruptedFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
4645
Pair<Path, String> tempFile = generateTempFile();
4746
Path destinationDir = Files.createTempDirectory("dice-where");
48-
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
49-
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
50-
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
51-
aResponse().withStatus(HttpStatus.SC_OK)
52-
.withHeader("Etag", "aaa")
53-
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
54-
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
55-
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
56-
.willReturn(aResponse().withBody(
57-
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
47+
IpInfoSiteSource ipInfoSiteSource =
48+
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
49+
wireMockRule.stubFor(
50+
WireMock.head(UrlPattern.ANY)
51+
.willReturn(
52+
aResponse()
53+
.withStatus(HttpStatus.SC_OK)
54+
.withHeader("Etag", "aaa")
55+
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
56+
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
57+
wireMockRule.stubFor(
58+
WireMock.get(UrlPattern.ANY)
59+
.willReturn(
60+
aResponse()
61+
.withBody(
62+
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
5863

5964
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
6065
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
@@ -68,72 +73,100 @@ public void corruptedFilePreexistingSet() throws IOException, NoSuchAlgorithmExc
6873
Pair<Path, String> existingFile = generateTempFile();
6974
Path destinationDir = Files.createTempDirectory("dice-where");
7075
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
71-
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
72-
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
73-
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
74-
aResponse().withStatus(HttpStatus.SC_OK)
75-
.withHeader("Etag", "aaa")
76-
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
77-
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
78-
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
79-
.willReturn(aResponse().withBody(
80-
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
76+
IpInfoSiteSource ipInfoSiteSource =
77+
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
78+
wireMockRule.stubFor(
79+
WireMock.head(UrlPattern.ANY)
80+
.willReturn(
81+
aResponse()
82+
.withStatus(HttpStatus.SC_OK)
83+
.withHeader("Etag", "aaa")
84+
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
85+
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
86+
wireMockRule.stubFor(
87+
WireMock.get(UrlPattern.ANY)
88+
.willReturn(
89+
aResponse()
90+
.withBody(
91+
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
8192

8293
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
8394
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
8495
assertNotEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
8596
assertEquals(Files.list(destinationDir).count(), 1);
8697
assertFalse(Files.exists(destinationDir.resolve("file.mdb")));
87-
assertTrue(Arrays.equals(Files.readAllBytes(existingFile.getLeft()),
88-
Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
98+
assertTrue(
99+
Arrays.equals(
100+
Files.readAllBytes(existingFile.getLeft()),
101+
Files.readAllBytes(Files.list(destinationDir).findFirst().get())));
89102
}
90103

91104
@Test
92105
public void goodFileEmptyPreexistingSet() throws IOException, NoSuchAlgorithmException {
93106
Pair<Path, String> tempFile = generateTempFile();
94107
Path destinationDir = Files.createTempDirectory("dice-where");
95-
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
96-
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
97-
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
98-
aResponse().withStatus(HttpStatus.SC_OK)
99-
.withHeader("Etag", tempFile.getRight())
100-
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
101-
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
102-
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
103-
.willReturn(aResponse().withBody(
104-
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
108+
IpInfoSiteSource ipInfoSiteSource =
109+
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
110+
wireMockRule.stubFor(
111+
WireMock.head(UrlPattern.ANY)
112+
.willReturn(
113+
aResponse()
114+
.withStatus(HttpStatus.SC_OK)
115+
.withHeader("Etag", tempFile.getRight())
116+
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
117+
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
118+
wireMockRule.stubFor(
119+
WireMock.get(UrlPattern.ANY)
120+
.willReturn(
121+
aResponse()
122+
.withBody(
123+
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
105124

106125
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
107126
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
108127
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
109128
assertEquals(1, Files.list(destinationDir).count());
110-
assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
111-
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
129+
assertTrue(
130+
Arrays.equals(
131+
Files.readAllBytes(tempFile.getLeft()),
132+
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
112133
}
113134

114135
@Test
115136
public void goodFilePreexistingSet() throws IOException, NoSuchAlgorithmException {
116137
Pair<Path, String> tempFile = generateTempFile();
117138
Pair<Path, String> existingFile = generateTempFile();
139+
118140
Path destinationDir = Files.createTempDirectory("dice-where");
119141
Files.copy(existingFile.getLeft(), destinationDir.resolve("existingFile.mdb"));
120-
IpInfoSiteSource ipInfoSiteSource = new IpInfoSiteSource(
121-
new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
122-
wireMockRule.stubFor(WireMock.head(UrlPattern.ANY).willReturn(
123-
aResponse().withStatus(HttpStatus.SC_OK)
124-
.withHeader("Etag", tempFile.getRight())
125-
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
126-
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
127-
wireMockRule.stubFor(WireMock.get(UrlPattern.ANY)
128-
.willReturn(aResponse().withBody(
129-
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
130142

143+
wireMockRule.stubFor(
144+
WireMock.head(UrlPattern.ANY)
145+
.willReturn(
146+
aResponse()
147+
.withStatus(HttpStatus.SC_OK)
148+
.withHeader("Etag", tempFile.getRight())
149+
.withHeader("Content-Length", Long.toString(TEST_FILE_SIZE))
150+
.withHeader("Last-Modified", "Thu, 01 Dec 1994 16:00:00 GMT")));
151+
152+
wireMockRule.stubFor(
153+
WireMock.get(UrlPattern.ANY)
154+
.willReturn(
155+
aResponse()
156+
.withBody(
157+
IOUtils.toByteArray(new FileInputStream(tempFile.getLeft().toFile())))));
158+
159+
IpInfoSiteSource ipInfoSiteSource =
160+
new IpInfoSiteSource(new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb"));
131161
FileInfo fileInfo = ipInfoSiteSource.fileInfo();
162+
132163
ipInfoSiteSource.produce(new LocalFileAcceptor(destinationDir.resolve("file.mdb")), false);
133164
assertEquals(tempFile.getRight().toLowerCase(), fileInfo.getMd5Checksum().stringFormat());
134165
assertEquals(2, Files.list(destinationDir).count());
135-
assertTrue(Arrays.equals(Files.readAllBytes(tempFile.getLeft()),
136-
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
166+
assertTrue(
167+
Arrays.equals(
168+
Files.readAllBytes(tempFile.getLeft()),
169+
Files.readAllBytes(destinationDir.resolve("file.mdb"))));
137170
}
138171

139172
private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorithmException {
@@ -145,4 +178,4 @@ private Pair<Path, String> generateTempFile() throws IOException, NoSuchAlgorith
145178
String hex = (new HexBinaryAdapter()).marshal(md.digest(contents));
146179
return Pair.of(tempFile, hex);
147180
}
148-
}
181+
}

‎dice-where-downloader-lib/src/test/java/technology/dice/dicewhere/downloader/stream/StreamWithMD5DecoratorTest.java

+81
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
package technology.dice.dicewhere.downloader.stream;
2+
3+
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
4+
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
5+
6+
import com.github.tomakehurst.wiremock.client.WireMock;
7+
import com.github.tomakehurst.wiremock.junit.WireMockRule;
8+
import com.github.tomakehurst.wiremock.matching.UrlPattern;
9+
import java.io.FileInputStream;
10+
import java.io.IOException;
11+
import java.net.HttpURLConnection;
12+
import java.net.URISyntaxException;
13+
import java.net.URL;
14+
import java.nio.charset.Charset;
15+
import java.nio.file.Path;
16+
import java.security.NoSuchAlgorithmException;
17+
import junit.framework.TestCase;
18+
import org.apache.commons.io.IOUtils;
19+
import org.junit.BeforeClass;
20+
import org.junit.ClassRule;
21+
import org.junit.Test;
22+
import org.junit.internal.runners.JUnit4ClassRunner;
23+
import org.junit.runner.RunWith;
24+
25+
@RunWith(JUnit4ClassRunner.class)
26+
public class StreamWithMD5DecoratorTest extends TestCase {
27+
28+
private static final String PATH = "/maxmind/maxmind-city-1.zip";
29+
30+
@ClassRule static WireMockRule wireMockRule = new WireMockRule(wireMockConfig().dynamicPort());
31+
32+
@BeforeClass
33+
public static void beforeClass() {
34+
wireMockRule.start();
35+
}
36+
37+
@Test
38+
public void shouldSuccessfullyReadAndCalculateDigestOfStream()
39+
throws IOException, NoSuchAlgorithmException, URISyntaxException {
40+
Path path = Path.of(getClass().getResource(PATH).toURI());
41+
StreamWithMD5Decorator is = StreamWithMD5Decorator.of(new FileInputStream(path.toFile()));
42+
43+
// Exhaust stream for the complete hash digest
44+
byte[] buffer = new byte[8192];
45+
while ((is.read(buffer)) != -1) {}
46+
47+
String first = is.md5().stringFormat();
48+
IOUtils.toString(is, Charset.defaultCharset());
49+
50+
// Assert the Stream Hash before and after
51+
assertEquals(first, is.md5().stringFormat());
52+
assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
53+
}
54+
55+
@Test
56+
public void shouldSuccessfullyReadAndCalculateDigestOfStreamFromHttp()
57+
throws IOException, NoSuchAlgorithmException, URISyntaxException {
58+
Path path = Path.of(getClass().getResource(PATH).toURI());
59+
60+
wireMockRule.stubFor(
61+
WireMock.get(UrlPattern.ANY)
62+
.willReturn(
63+
aResponse().withBody(IOUtils.toByteArray(new FileInputStream(path.toFile())))));
64+
65+
URL url = new URL("http://localhost:" + wireMockRule.port() + "/data/file.mdb");
66+
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
67+
68+
StreamWithMD5Decorator is = StreamWithMD5Decorator.of(connection.getInputStream());
69+
// Exhaust stream for the complete hash digest
70+
byte[] buffer = new byte[8192];
71+
while ((is.read(buffer)) != -1) {}
72+
73+
String first = is.md5().stringFormat();
74+
// Read from the stream
75+
IOUtils.toString(is, Charset.defaultCharset());
76+
77+
// Assert the Stream Hash before and after
78+
assertEquals(first, is.md5().stringFormat());
79+
assertEquals(first, "9c7dd68c8352f1c59a33efe0dca04f06");
80+
}
81+
}

‎dice-where-downloader-lib/src/test/resources/maxmind/maxmind-city-1.zip

1.86 KB
Binary file not shown.

0 commit comments

Comments
 (0)