Skip to content

Commit 5956c5b

Browse files
blabadiblabadi
authored
Merge pull request #153 from overture-stack/rc/1.1.0
Features: Read existing config and set them as defaults Skip unchangeable configurations Pull docker images before deploy to avoid timeouts Backup existing config.yaml before saving new configurations. Changes: Add canari credits in readme.md Update docs links upgrade dms ui to 1.0.2 Fixes: Fix dms home path in the wrapper script (requires downloading new script) Fix user jwt duration configuration setting
2 parents cb13e61 + 3e00c00 commit 5956c5b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+529
-246
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,7 @@ build/
3232

3333
### VS Code ###
3434
.vscode/
35+
36+
37+
## others
38+
run.sh

‎README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
Overture Data Management System
33

44

5-
## Developmenet
5+
## Development
66
### Configure remote docker daemon control
77
It is possible to run the dms locally while controlling a remote docker engine.
88
1. Port forward the docker.sock [Forwarding the Docker Socket over SSH](https://medium.com/@dperny/forwarding-the-docker-socket-over-ssh-e6567cfab160)
@@ -44,3 +44,9 @@ Sometimes, if the reserved/limit memory is too low, a container will get killed
4444
## useful commands:
4545
- `docker service ps --no-trunc {serviceName}`
4646
- `journalctl -u docker.service | tail -n 50 `
47+
48+
## Acknowledgements
49+
50+
DMS development supported by:
51+
52+
[![Canarie logo](canarie-logo.png)](https://canarie.ca)

‎canarie-logo.png‎

5.72 KB
Loading

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
</parent>
1111
<groupId>bio.overture</groupId>
1212
<artifactId>dms</artifactId>
13-
<version>1.0.0</version>
13+
<version>1.1.0</version>
1414
<name>dms</name>
1515
<description>Overture Data Management System</description>
1616

‎src/main/bin/dms-docker‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ function main {
175175
docker_opts="$docker_opts -p 5005:5005"
176176
set -x
177177
fi
178-
docker_opts="$docker_opts -e DOCKER_RUNAS=true -e DMS_HOME_HOST_PATH=$DMS_HOME"
178+
docker_opts="$docker_opts -e DOCKER_RUNAS=true -e DOCKER_DMSHOMEHOSTPATH=$DMS_HOME -e DOCKER_TAG=$dms_version"
179179
# create dms-swarm-network if it doesn't exist, this is important for first run
180180
# since the dms uses that in its run options
181181
found_network=$(docker network ls --filter name=${NETWORK_NAME} --format="{{ .Name }}")

‎src/main/java/bio/overture/dms/cli/DmsConfigStore.java‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66

77
import bio.overture.dms.core.model.dmsconfig.DmsConfig;
88
import bio.overture.dms.core.util.ObjectSerializer;
9+
import bio.overture.dms.swarm.properties.DockerProperties;
10+
11+
import java.io.File;
912
import java.nio.file.Path;
1013
import java.nio.file.Paths;
14+
import java.text.SimpleDateFormat;
15+
import java.util.Date;
1116
import java.util.Optional;
1217
import java.util.function.Function;
1318
import lombok.NonNull;
@@ -25,9 +30,13 @@ public class DmsConfigStore {
2530
/** Dependencies */
2631
private final ObjectSerializer yamlSerializer;
2732

33+
private DockerProperties properties;
34+
2835
@Autowired
29-
public DmsConfigStore(@NonNull ObjectSerializer yamlSerializer) {
36+
public DmsConfigStore(
37+
@NonNull ObjectSerializer yamlSerializer, @NonNull DockerProperties properties) {
3038
this.yamlSerializer = yamlSerializer;
39+
this.properties = properties;
3140
}
3241

3342
@SneakyThrows
@@ -56,19 +65,31 @@ public Optional<DmsConfig> findStoredConfig() {
5665
return Optional.empty();
5766
}
5867

59-
public void save(DmsConfig dmsConfig) {
68+
private void save(DmsConfig dmsConfig) {
6069
yamlSerializer.serializeToFile(dmsConfig, getDmsConfigFilePath().toFile());
6170
}
6271

6372
public void apply(Function<DmsConfig, DmsConfig> transformation) {
64-
val storedDmsConfig = findStoredConfig().orElse(null);
65-
66-
// TODO: ideally, this is how a null is treated
67-
// findStoredConfig()
68-
// .map(transformation::apply)
69-
// .ifPresent(this::save);
70-
73+
val storedDmsConfig =
74+
findStoredConfig()
75+
// this is only needed when running locally (make sure to set the application properties
76+
// correctly)
77+
// in real scenarios it will be passed from the dms-docker script which is read from the
78+
// initial
79+
// config file.
80+
.orElse(
81+
yamlSerializer.deserializeToObject(
82+
"version: " + properties.getTag(), DmsConfig.class));
7183
val dmsConfig = transformation.apply(storedDmsConfig);
84+
backupExistingConfig(storedDmsConfig);
7285
save(dmsConfig);
7386
}
87+
88+
@SneakyThrows
89+
private void backupExistingConfig(DmsConfig storedDmsConfig) {
90+
val df = new SimpleDateFormat("yyyyMMddHHmm");
91+
File f = new File(getDmsConfigFilePath() + ".backup-" + df.format(new Date()));
92+
f.createNewFile();
93+
yamlSerializer.serializeToFile(storedDmsConfig, f);
94+
}
7495
}

‎src/main/java/bio/overture/dms/cli/command/cluster/ClusterDestroyCommand.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,7 @@ private boolean confirmVolumesDeletion() {
8181
if (askQuestion) {
8282
confirmedVolumeDestruction =
8383
questionFactory
84-
.newSingleQuestion(
85-
WARNING,
86-
Boolean.class,
87-
CONFIRM_DESTROY,
88-
true,
89-
false)
84+
.newSingleQuestion(WARNING, Boolean.class, CONFIRM_DESTROY, true, false)
9085
.getAnswer();
9186
}
9287
if (confirmedVolumeDestruction) {

‎src/main/java/bio/overture/dms/cli/command/config/ConfigBuildCommand.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ public ConfigBuildCommand(
4242
public Integer call() throws Exception {
4343
t.print(PRE_REQ_NOTE);
4444
t.printStatusLn("Starting interactive configuration...");
45-
// TODO: Fix this so that the storedDmsConfig is input into the buildDmsConfig method
4645
dmsConfigStore.apply(dmsQuestionnaire::buildDmsConfig);
4746
t.printStatusLn(CONFIGURATION_SAVED_MSG, dmsConfigStore.getDmsConfigFilePath());
4847
return 0;

‎src/main/java/bio/overture/dms/cli/model/Constants.java‎

Lines changed: 60 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,60 @@ public final class Constants {
1010
public static final String CONFIG_FILE_NAME = "config.yaml";
1111

1212
public static final class CommandsQuestions {
13-
public static final String CONFIRM_DESTROY = "Are you sure you want to destroy the volumes for all services, all data will be lost and This is IRREVERSIBLE ? ";
13+
public static final String CONFIRM_DESTROY =
14+
"Are you sure you want to destroy the volumes for all services, all data will be lost and This is IRREVERSIBLE ? ";
1415
}
16+
1517
public static final class SongQuestions {
1618
public static String PASSWORD = "What should the SONG database password be?";
19+
public static final String PASSWORD_CONFIGURED_SONG_DB = "A password is already configured for SONG db.";
20+
}
21+
22+
public static final class DockerImagesConstants {
23+
public static final String OVERTURE_EGO_UI = "overture/ego-ui";
24+
public static final String OVERTURE_ARRANGER_UI = "overture/arranger-ui";
25+
public static final String OVERTURE_DMS_UI = "overture/dms-ui";
26+
public static final String OVERTURE_ARRANGER_SERVER = "overture/arranger-server";
27+
public static final String OVERTURE_EGO = "overture/ego";
28+
public static final String OVERTURE_SONG_SERVER = "overture/song-server";
29+
public static final String OVERTURE_SCORE_SERVER = "overture/score-server";
30+
public static final String POSTGRES = "postgres";
31+
public static final String MINIO_MINIO = "minio/minio";
32+
public static final String DOCKER_ELASTIC_CO_ELASTICSEARCH_ELASTICSEARCH =
33+
"docker.elastic.co/elasticsearch/elasticsearch";
34+
public static final String GHCR_IO_OVERTURE_STACK_DMS_GATEWAY =
35+
"ghcr.io/overture-stack/dms-gateway";
36+
public static final String GHCR_IO_OVERTURE_STACK_DMS_GATEWAY_SECURE =
37+
"ghcr.io/overture-stack/dms-gateway-secure";
38+
public static final String GHCR_IO_OVERTURE_STACK_MAESTRO = "ghcr.io/overture-stack/maestro";
39+
40+
public static final String EGO_UI_TAG = "4.2.0";
41+
public static final String ARRANGER_UI_TAG = "2.12.3";
42+
public static final String ARRANGER_SERVER_TAG = "2.12.3";
43+
public static final String SCORE_SERVER_TAG = "5.3.0";
44+
public static final String EGO_TAG = "4.4.0";
45+
public static final String SONG_SERVER_TAG = "4.5.0";
46+
public static final String POSTGRES_TAG = "11.1";
47+
public static final String MINIO_TAG = "RELEASE.2018-05-11T00-29-24Z";
48+
public static final String ES_TAG = "7.6.0";
49+
public static final String DMS_GATEWAY_TAG = "1.0.0";
50+
public static final String MAESTRO_TAG = "3.8.0";
51+
public static final String DMS_UI_TAG = "1.0.2";
1752
}
1853

1954
public static final class MESSAGES {
55+
56+
public static final String CHECK_DOCKER_IMAGES_MSG =
57+
"Checking if docker images exists & downloading missing ones, this can take few minutes depending on the connection speed.. ";
58+
public static final String CHECK_COMPLETED = "Check completed.";
59+
2060
public static final String CONFIGURATION_SAVED_MSG =
21-
"Your configuration file was successfully saved to: /root/.dms/config.yaml\n"
61+
"Your configuration file was successfully saved to: /root/.dms/config.yaml\n"
2262
+ "\n"
2363
+ "You may now deploy your configuration to your cluster. For instructions, see:\n"
2464
+ "https://overture.bio/documentation/dms/installation/deploy/\n";
2565
public static final String PRE_REQ_NOTE =
26-
"*****************************************************************************************************\n"
66+
"*****************************************************************************************************\n"
2767
+ "!!! NOTE !!!\n"
2868
+ "\n"
2969
+ " Before starting, make sure you have completed all prerequisite setup steps here:\n"
@@ -32,13 +72,13 @@ public static final class MESSAGES {
3272
+ "*****************************************************************************************************";
3373

3474
public static final String POST_DEPLOYMENT_MSG =
35-
"*****************************************************************************************************\n"
75+
"*****************************************************************************************************\n"
3676
+ "!!! NOTE !!!\n"
3777
+ "\n"
3878
+ " Before using the DMS platform, please complete post-deployment verification \n"
3979
+ " and configuration steps required to check the health of your deployment. For \n"
4080
+ " instructions, see:\n"
41-
+ " https://overture.bio/documentation/dms/installation/deploy-and-verify/\n"
81+
+ " https://overture.bio/documentation/dms/installation/verify/\n"
4282
+ "\n"
4383
+ "*****************************************************************************************************";
4484
}
@@ -55,26 +95,26 @@ public static final class GATEWAY {
5595

5696
public static final class GuidesURLS {
5797
public static final String HELP_HEADER_GUIDE_URLS =
58-
"\n"
98+
"\n"
5999
+ "Installation Guide: https://overture.bio/documentation/dms/installation/"
60100
+ "\n"
61101
+ "User Guide: https://overture.bio/documentation/dms/user-guide/"
62102
+ "\n"
63103
+ "\n";
64104
public static final String GUIDE_DMSUI =
65-
"Guide: https://overture.bio/documenation/dms/installation/configuration/dms-ui";
105+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-dms-ui";
66106
public static final String GUIDE_MAESTRO =
67-
"Guide: https://overture.bio/documenation/dms/installation/configuration/maestro";
107+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-maestro";
68108
public static final String DEPLOYMENT_MODE =
69-
"Guide: https://overture.bio/documenation/dms/installation/configuration/deployment-mode";
109+
"Guide: https://overture.bio/documentation/dms/installation/configuration/#decide-local-or-server-deployment";
70110
public static final String GUIDE_EGO =
71-
"Guide: https://overture.bio/documenation/dms/installation/configuration/ego";
111+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-ego";
72112
public static final String GUIDE_SONG =
73-
"Guide: https://overture.bio/documenation/dms/installation/configuration/song";
113+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-song";
74114
public static final String GUIDE_SCORE =
75-
"Guide: https://overture.bio/documenation/dms/installation/configuration/score";
115+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-score";
76116
public static final String GUIDE_ES =
77-
"Guide: https://overture.bio/documenation/dms/installation/configuration/elasticsearch";
117+
"Guide: https://overture.bio/documentation/dms/installation/configuration/configure-dms/#configure-elasticsearch";
78118
}
79119

80120
public static final class ScoreQuestions {
@@ -95,6 +135,7 @@ public static final class ScoreQuestions {
95135
+ "If no, you must enter them in the subsequent questions.";
96136
public static final String MINIO_ACCESS_KEY = "What should the MinIO access key be?";
97137
public static final String MINIO_SECRET_KEY = "What should the MinIO secret key be?";
138+
public static final String YOU_HAVE_ALREADY_CONFIGURED_AN_S3 = "You have already configured an S3 object storage service, do you want to keep the same configurations ?";
98139
}
99140

100141
public static final class EgoQuestions {
@@ -110,6 +151,8 @@ public static final class EgoQuestions {
110151
public static final String PASSWORD = "What should the EGO database password be?";
111152
public static final String WHAT_IS_THE_S_CLIENT_ID = "What is the %s client ID?";
112153
public static final String WHAT_IS_THE_S_CLIENT_SECRET = "What is the %s client secret?";
154+
public static final String YOU_HAVE_ALREADY_CONFIGURED_THE_SSO_PROVIDERS = "You have already configured the SSO providers, do you want to keep the same configurations ?";
155+
public static final String PASSWORD_CONFIGURED_EGO_DB = "A password is already configured for EGO db.";
113156
}
114157

115158
@UtilityClass
@@ -125,20 +168,21 @@ public class DmsUiQuestions {
125168
public static final String ALIAS =
126169
"What is the Elasticsearch alias name you will configure in Arranger (to be referenced by DMS UI and ALSO must match the alias name previously supplied for Maestro) be? ";
127170
public static final String ARRANGER_QUESTIONS_NOTE =
128-
"\n*****************************************************************************************************\n"
171+
"\n*****************************************************************************************************\n"
129172
+ "!!! NOTE !!!\n"
130173
+ "\n"
131174
+ " The next 3 fields (Arranger Project ID, Project Name, Elasticsearch AliasName are \n"
132175
+ " required when you create your project in the Arranger administrative UI after \n"
133176
+ " deployment. The values you use MUST match the ones you supply here for the \n"
134177
+ " DMS UI configuration. The DMS UI interacts with Arranger and expects the same \n"
135178
+ " values you input here. For instructions on adding an Arranger project, see:\n"
136-
+ " https://overture.bio/documentation/dms/installation/deploy-and-verify/\n"
137-
179+
+ " https://overture.bio/documentation/dms/installation/verify/#add-project-to-arranger-ui\n"
138180
+ "*****************************************************************************************************\n";
139181
}
140182

141183
public static final class MaestroQuestions {
184+
public static final String PASSWORD_CONFIGURED_ELASTICSEARCH = "A password is already configured for Elasticsearch";
185+
142186
public static final String ES_PASSWORD =
143187
"Elasticsearch provides a superuser with default username 'elastic'. What should the superuser's password be?";
144188
public static String ALIAS =

‎src/main/java/bio/overture/dms/cli/questionnaire/ArrangerQuestionnaire.java‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,24 +15,19 @@
1515
@Component
1616
public class ArrangerQuestionnaire {
1717

18-
private final QuestionFactory questionFactory;
19-
2018
@Autowired
21-
public ArrangerQuestionnaire(@NonNull QuestionFactory questionFactory) {
22-
this.questionFactory = questionFactory;
23-
}
19+
public ArrangerQuestionnaire() { }
2420

25-
public ArrangerConfig buildConfig(ClusterRunModes clusterRunMode, GatewayConfig gatewayConfig) {
21+
public ArrangerConfig buildConfig(GatewayConfig gatewayConfig) {
2622

2723
val info =
28-
resolveServiceConnectionInfo(
29-
clusterRunMode, gatewayConfig, questionFactory, ARRANGER_SERVER.toString(), 5050);
24+
resolveServiceConnectionInfo(gatewayConfig,
25+
ARRANGER_SERVER.toString(),
26+
5050);
3027

3128
val uiHostInfo =
3229
resolveServiceConnectionInfo(
33-
clusterRunMode,
3430
gatewayConfig,
35-
questionFactory,
3631
ARRANGER_UI.toString(),
3732
ArrangerConfig.ArrangerUIConfig.DEFAULT_PORT);
3833
return ArrangerConfig.builder()

0 commit comments

Comments
 (0)