Skip to content

Commit 39caae5

Browse files
alismanhaynescdonursumeruklinealeBryan Lai
authored
Clickhouse mode of study view (#11224)
Add Clickhouse implementation of study view endpoints (off by default) with legacy api intact --------- Co-authored-by: haynescd <donhaynes@gmail.com> --------- Co-authored-by: haynescd <haynescd@chop.edu> Co-authored-by: Charles Haynes <33608920+haynescd@users.noreply.github.com> Co-authored-by: Onur Sumer <s.onur.sumer@gmail.com> Co-authored-by: Neel Kuila <neelk1123@gmail.com> Co-authored-by: Bryan Lai <laib1@mskcc.org> Co-authored-by: Zhaoyuan (Ryan) Fu <fuzhaoyuan@gmail.com> Co-authored-by: Qi-Xuan Lu <qlu@carisls.com> Co-authored-by: Bryan Lai <56288120+gblaih@users.noreply.github.com> Co-authored-by: Gaofei Zhao <15748980+dippindots@users.noreply.github.com> Co-authored-by: Robert Sheridan <sheridan@cbio.mskcc.org> Co-authored-by: Zain Nasir <49543216+zainasir@users.noreply.github.com> Co-authored-by: Zain Nasir <zainasir1999@gmail.com> Co-authored-by: haynescd <donhaynes@gmail.com>
1 parent be86bae commit 39caae5

File tree

137 files changed

+26236
-431
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

137 files changed

+26236
-431
lines changed

‎.circleci/config.yml

+147
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,134 @@ jobs:
223223
DOCKER_IMAGE_SESSION_SERVICE: cbioportal/session-service:0.5.0
224224
FRONTEND_TEST_DO_NOT_LOAD_EXTERNAL_FRONTEND: true
225225

226+
pull_cbioportal_test_codebase:
227+
machine:
228+
image: ubuntu-2204:2024.08.1
229+
resource_class: medium
230+
working_directory: /tmp/repos
231+
steps:
232+
- run:
233+
name: Checkout cbioportal/cbioportal-test
234+
environment:
235+
TEST_REPO_URL: https://github.com/cBioPortal/cbioportal-test
236+
command: |
237+
git clone ${TEST_REPO_URL}
238+
- persist_to_workspace:
239+
root: /tmp/repos
240+
paths:
241+
- cbioportal-test
242+
243+
pull_cbioportal_frontend_codebase:
244+
machine:
245+
image: ubuntu-2204:2024.08.1
246+
resource_class: medium
247+
working_directory: /tmp/repos
248+
steps:
249+
- run:
250+
name: Checkout cbioportal/cbioportal-frontend
251+
environment:
252+
FRONTEND_REPO_URL: https://github.com/cBioPortal/cbioportal-frontend.git
253+
FRONTEND_REPO_BRANCH: master
254+
command: |
255+
git clone -b ${FRONTEND_REPO_BRANCH} --single-branch ${FRONTEND_REPO_URL}
256+
- persist_to_workspace:
257+
root: /tmp/repos
258+
paths:
259+
- cbioportal-frontend
260+
261+
build_push_image:
262+
machine:
263+
image: ubuntu-2204:2024.08.1
264+
resource_class: medium
265+
working_directory: /tmp/repos
266+
steps:
267+
- attach_workspace:
268+
at: /tmp/repos
269+
- checkout:
270+
path: /tmp/repos/cbioportal
271+
- run:
272+
name: Build cBioPortal docker image
273+
environment:
274+
DOCKER_REPO: cbioportal/cbioportal-dev
275+
command: |
276+
export DOCKER_TAG=$CIRCLE_SHA1
277+
URL="https://hub.docker.com/v2/repositories/cbioportal/cbioportal-dev/tags/$DOCKER_TAG-web-shenandoah"
278+
TAG_FOUND=$(curl -s $URL | jq -r .name)
279+
if [ $TAG_FOUND = "$DOCKER_TAG-web-shenandoah" ]; then
280+
echo "Image already exists. Skipping build step!"
281+
exit 0
282+
fi
283+
cd cbioportal-test
284+
./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=true --skip_web_and_data=true
285+
EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?)
286+
if [ $EXISTS -eq 0 ]; then
287+
echo "Build succeeded!"
288+
else
289+
echo "Build failed!"
290+
exit 1
291+
fi
292+
293+
run_api_tests:
294+
machine:
295+
image: ubuntu-2204:2024.08.1
296+
docker_layer_caching: true
297+
resource_class: large
298+
working_directory: /tmp/repos
299+
steps:
300+
- attach_workspace:
301+
at: /tmp/repos
302+
- checkout:
303+
path: /tmp/repos/cbioportal
304+
- run:
305+
name: Instantiate a cbioportal instance
306+
environment:
307+
DOCKER_REPO: cbioportal/cbioportal-dev
308+
APP_CLICKHOUSE_MODE: "true"
309+
command: |
310+
cd cbioportal-test
311+
export DOCKER_IMAGE_CBIOPORTAL=$DOCKER_REPO:$CIRCLE_SHA1-web-shenandoah
312+
nohup ./scripts/docker-compose.sh >> /tmp/repos/docker-compose-logs.txt 2>&1 &
313+
- run:
314+
name: Wait for cbioportal to be live at localhost
315+
command: |
316+
cd cbioportal-test
317+
./utils/check-connection.sh --url=localhost:8080
318+
- run:
319+
name: Confirm cbioportal config matches PR
320+
command: |
321+
cd cbioportal
322+
echo "Matching gitCommitId..."
323+
INSTANCE_COMMIT_ID=$(curl -s http://localhost:8080/api/info | jq -r '.["gitCommitId"]')
324+
PR_COMMIT_ID=$CIRCLE_SHA1
325+
if [ "$INSTANCE_COMMIT_ID" = "$PR_COMMIT_ID" ]; then
326+
echo "gitCommitId successfully matched!"
327+
echo "cBioPortal is ready:"
328+
curl -s http://localhost:8080/api/info | jq
329+
exit 0
330+
else
331+
echo "gitCommitIds do not match!"
332+
echo "Instance Commit ID: $INSTANCE_COMMIT_ID"
333+
echo "PR Commit ID: $PR_COMMIT_ID"
334+
exit 1
335+
fi
336+
- run:
337+
name: Run API Tests
338+
environment:
339+
API_TEST_HOST: http://localhost:8080
340+
BACKEND_ROOT: /tmp/repos/cbioportal
341+
command: |
342+
cd cbioportal-frontend
343+
nvm install 15.2.1
344+
nvm use 15.2.1
345+
npm install -g yarn@1.22.5
346+
yarn --ignore-engines
347+
yarn run apitests
348+
349+
- store_artifacts:
350+
path: /tmp/repos/cbioportal-test/web-metadata.json
351+
- store_artifacts:
352+
path: /tmp/repos/docker-compose-logs.txt
353+
226354

227355
workflows:
228356
version: 2
@@ -238,3 +366,22 @@ workflows:
238366
- build_backend
239367
- pull_frontend_codebase
240368
- install_yarn
369+
api_tests:
370+
jobs:
371+
- pull_cbioportal_test_codebase
372+
- pull_cbioportal_frontend_codebase
373+
- wait_for_approval:
374+
type: approval
375+
requires:
376+
- pull_cbioportal_test_codebase
377+
- pull_cbioportal_frontend_codebase
378+
- build_push_image:
379+
context:
380+
- api-tests
381+
requires:
382+
- wait_for_approval
383+
- run_api_tests:
384+
context:
385+
- api-tests
386+
requires:
387+
- build_push_image

‎.github/workflows/sonarcloud.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
cp src/main/resources/application.properties.EXAMPLE src/main/resources/application.properties
3535
- name: Build and analyze
3636
env:
37-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
37+
SONAR_TOKEN: de1b5cc660cd210dde840f492c371da6cc801763
3838
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3939
run: mvn clean install verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
4040
- name: 'Add host.testcontainers.internal to /etc/hosts'
@@ -45,6 +45,6 @@ jobs:
4545
mvn verify -Pintegration-test
4646
- name: Code Coverage
4747
env:
48-
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
48+
SONAR_TOKEN: de1b5cc660cd210dde840f492c371da6cc801763
4949
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5050
run: mvn sonar:sonar -Pcoverage

‎pom.xml

+42
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
<selenium.version>4.17.0</selenium.version>
100100
<sentry.version>7.1.0</sentry.version>
101101
<apache_httpclient.version>5.2.1</apache_httpclient.version>
102+
<clickhouse_testcontainer.version>1.19.7</clickhouse_testcontainer.version>
102103

103104

104105
<!-- No sure what these are for -->
@@ -354,6 +355,31 @@
354355
<scope>test</scope>
355356
</dependency>
356357

358+
<dependency>
359+
<groupId>com.clickhouse</groupId>
360+
<artifactId>clickhouse-jdbc</artifactId>
361+
<version>0.6.2</version>
362+
<!-- use uber jar with all dependencies included, change classifier
363+
to http for smaller jar -->
364+
<classifier>all</classifier>
365+
</dependency>
366+
<dependency>
367+
<groupId>org.testcontainers</groupId>
368+
<artifactId>clickhouse</artifactId>
369+
<version>${clickhouse_testcontainer.version}</version>
370+
<scope>test</scope>
371+
</dependency>
372+
<dependency>
373+
<groupId>org.mybatis.spring.boot</groupId>
374+
<artifactId>mybatis-spring-boot-test-autoconfigure</artifactId>
375+
<version>3.0.3</version>
376+
<scope>test</scope>
377+
</dependency>
378+
<dependency>
379+
<groupId>org.springframework.boot</groupId>
380+
<artifactId>spring-boot-testcontainers</artifactId>
381+
<scope>test</scope>
382+
</dependency>
357383
</dependencies>
358384

359385
<dependencyManagement>
@@ -418,9 +444,25 @@
418444
<includes>
419445
<include>cgds.sql</include>
420446
<include>migration.sql</include>
447+
<include>clickhouse/clickhouse.sql</include>
448+
<include>clickhouse/materialized_views.sql</include>
421449
</includes>
422450
</resource>
423451
</resources>
452+
<testResources>
453+
<testResource>
454+
<directory>src/main/resources/db-scripts</directory>
455+
<filtering>false</filtering>
456+
<includes>
457+
<include>clickhouse/clickhouse.sql</include>
458+
<include>clickhouse/materialized_views.sql</include>
459+
</includes>
460+
</testResource>
461+
<testResource>
462+
<directory>src/test/resources</directory>
463+
<filtering>false</filtering>
464+
</testResource>
465+
</testResources>
424466

425467
<plugins>
426468
<plugin>

‎src/main/java/org/cbioportal/PortalApplication.java

+2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
77
import org.springframework.context.annotation.PropertySource;
88
import org.springframework.context.annotation.PropertySources;
9+
import org.springframework.transaction.annotation.EnableTransactionManagement;
910

1011
@SpringBootApplication(exclude = {
1112
MongoAutoConfiguration.class,
@@ -18,6 +19,7 @@
1819
@PropertySource(ignoreResourceNotFound = true, value = "classpath:git.properties"),
1920
@PropertySource(ignoreResourceNotFound = true, value = "classpath:springdoc.properties")
2021
})
22+
@EnableTransactionManagement
2123
public class PortalApplication {
2224
public static void main(String[] args) {
2325
SpringApplication.run(PortalApplication.class, args);

‎src/main/java/org/cbioportal/WebAppConfig.java

+5
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.util.List;
44

5+
import org.cbioportal.web.ExecuterTimeInterceptor;
56
import org.cbioportal.web.util.InvolvedCancerStudyExtractorInterceptor;
67
import org.springframework.beans.factory.annotation.Value;
78
import org.springframework.context.annotation.Bean;
@@ -12,6 +13,7 @@
1213
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
1314
import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
1415
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
16+
import org.springframework.web.servlet.handler.WebRequestHandlerInterceptorAdapter;
1517

1618
// TODO Consider creating separate DispatcherServlets as in the original web.xml
1719
// See: https://stackoverflow.com/a/30686733/11651683
@@ -75,6 +77,9 @@ public HandlerInterceptor involvedCancerStudyExtractorInterceptor() {
7577
@Override
7678
public void addInterceptors(InterceptorRegistry registry) {
7779
registry.addInterceptor(involvedCancerStudyExtractorInterceptor());
80+
registry.addInterceptor(new WebRequestHandlerInterceptorAdapter(
81+
new ExecuterTimeInterceptor()
82+
)).addPathPatterns("/**");
7883
}
7984

8085
@Override

‎src/main/java/org/cbioportal/model/AlterationType.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.cbioportal.model;
22

33
public enum AlterationType {
4-
MUTATION,
5-
COPY_NUMBER_ALTERATION
4+
MUTATION_EXTENDED,
5+
COPY_NUMBER_ALTERATION,
6+
STRUCTURAL_VARIANT,
7+
GENERIC_ASSAY;
68
}

‎src/main/java/org/cbioportal/model/CaseListDataCount.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.cbioportal.model;
22

3-
public class CaseListDataCount {
3+
import java.io.Serializable;
4+
5+
public class CaseListDataCount implements Serializable {
46

57
private String label;
68
private String value;

‎src/main/java/org/cbioportal/model/ClinicalViolinPlotBoxData.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.cbioportal.model;
22

3-
public class ClinicalViolinPlotBoxData {
3+
import java.io.Serializable;
4+
5+
public class ClinicalViolinPlotBoxData implements Serializable {
46
private double whiskerLower;
57
private double whiskerUpper;
68
private double median;

‎src/main/java/org/cbioportal/model/ClinicalViolinPlotData.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package org.cbioportal.model;
22

3+
import java.io.Serializable;
34
import java.util.List;
45

5-
public class ClinicalViolinPlotData {
6+
public class ClinicalViolinPlotData implements Serializable {
67
private List<ClinicalViolinPlotRowData> rows;
78
private Double axisStart;
89
private Double axisEnd;

‎src/main/java/org/cbioportal/model/ClinicalViolinPlotIndividualPoint.java

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package org.cbioportal.model;
22

3-
public class ClinicalViolinPlotIndividualPoint {
3+
import java.io.Serializable;
4+
5+
public class ClinicalViolinPlotIndividualPoint implements Serializable {
46
private String sampleId;
57
private String studyId;
68
private double value;

‎src/main/java/org/cbioportal/model/ClinicalViolinPlotRowData.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package org.cbioportal.model;
22

3+
import java.io.Serializable;
34
import java.util.List;
45

5-
public class ClinicalViolinPlotRowData {
6+
public class ClinicalViolinPlotRowData implements Serializable {
67
private String category;
78
private Integer numSamples;
89
private List<Double> curveData;

‎src/main/java/org/cbioportal/model/GenericAssayDataCount.java

+7
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ public class GenericAssayDataCount implements Serializable {
77
private String value;
88
private Integer count;
99

10+
public GenericAssayDataCount() {}
11+
12+
public GenericAssayDataCount(String value, Integer count) {
13+
this.value = value;
14+
this.count = count;
15+
}
16+
1017
public String getValue() {
1118
return value;
1219
}

‎src/main/java/org/cbioportal/model/GenericAssayDataCountItem.java

+7
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ public class GenericAssayDataCountItem implements Serializable {
77
private String stableId;
88
private List<GenericAssayDataCount> counts;
99

10+
public GenericAssayDataCountItem() {}
11+
12+
public GenericAssayDataCountItem(String stableId, List<GenericAssayDataCount> counts) {
13+
this.stableId = stableId;
14+
this.counts = counts;
15+
}
16+
1017
public String getStableId() {
1118
return stableId;
1219
}

‎src/main/java/org/cbioportal/model/GenomicDataCount.java

+15
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,21 @@ public class GenomicDataCount implements Serializable {
1010
private Integer count;
1111
private Integer uniqueCount;
1212

13+
public GenomicDataCount() {}
14+
15+
public GenomicDataCount(String label, String value, Integer count) {
16+
this.label = label;
17+
this.value = value;
18+
this.count = count;
19+
}
20+
21+
public GenomicDataCount(String label, String value, Integer count, Integer uniqueCount) {
22+
this.label = label;
23+
this.value = value;
24+
this.count = count;
25+
this.uniqueCount = uniqueCount;
26+
}
27+
1328
public String getLabel() {
1429
return label;
1530
}

0 commit comments

Comments
 (0)