Skip to content

Commit f4cf4df

Browse files
o-alexo-alex
committed
[HWORKS-737] Deleted project leads to NPE when processing SearchFSCommand (#1548)
1 parent ccb1aa9 commit f4cf4df

File tree

6 files changed

+40
-40
lines changed

6 files changed

+40
-40
lines changed

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSCommandExecutor.java‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
import io.hops.hopsworks.persistence.entity.commands.search.SearchFSCommand;
3333
import io.hops.hopsworks.persistence.entity.commands.search.SearchFSCommandHistory;
3434
import io.hops.hopsworks.persistence.entity.commands.search.SearchFSCommandOp;
35-
import io.hops.hopsworks.persistence.entity.project.Project;
3635
import io.hops.hopsworks.restutils.RESTCodes;
3736

3837
import javax.annotation.PostConstruct;
@@ -138,7 +137,7 @@ public void process() {
138137
try {
139138
processInt();
140139
} catch (Exception t) {
141-
LOGGER.log(Level.INFO, "Command processing failed with error:", t.getStackTrace());
140+
LOGGER.log(Level.INFO, "Command processing failed with error", t);
142141
}
143142
}
144143
schedule();
@@ -160,34 +159,32 @@ private void processInt() throws CommandException {
160159
return;
161160
}
162161
Set<Long> updatingDocs = updatingCommands.stream().map(this::getDocId).collect(Collectors.toSet());
163-
Map<Integer, Project> updatingProjects = updatingCommands.stream()
164-
.collect(Collectors.toMap(c -> c.getProject().getId(), Command::getProject, (p1, p2) -> p1));
162+
Set<Integer> updatingProjects = updatingCommands.stream().map(Command::getProjectId).collect(Collectors.toSet());
165163

166164
List<SearchFSCommand> cleaningCommands = commandFacade.findByQuery(queryByStatus(CommandStatus.CLEANING));
167165
active += cleaningCommands.size();
168166
if (active >= maxOngoing) {
169167
return;
170168
}
171169
Set<Long> cleaningDocs = cleaningCommands.stream().map(this::getDocId).collect(Collectors.toSet());
172-
updatingProjects.putAll(cleaningCommands.stream().collect(Collectors.toMap(c -> c.getProject().getId(),
173-
Command::getProject, (existingP, newP) -> existingP)));
170+
updatingProjects.addAll(cleaningCommands.stream().map(Command::getProjectId).collect(Collectors.toSet()));
174171

175172
Map<Integer, SearchFSCommand> toDeleteProjects = commandFacade.findByQuery(queryDeletedProjects(CommandStatus.NEW))
176-
.stream().collect(Collectors.toMap(c -> c.getProject().getId(), c -> c, (existingC, newC) -> existingC));
173+
.stream().collect(Collectors.toMap(Command::getProjectId, c -> c, (existingC, newC) -> existingC));
177174
Map<Integer, SearchFSCommand> deletingProjects
178175
= commandFacade.findByQuery(queryDeletedProjects(CommandStatus.CLEANING))
179-
.stream().collect(Collectors.toMap(c -> c.getProject().getId(), c -> c, (existingC, newC) -> existingC));
176+
.stream().collect(Collectors.toMap(Command::getProjectId, c -> c, (existingC, newC) -> existingC));
180177

181178
List<SearchFSCommand> failedCommands = commandFacade.findByQuery(queryByStatus(CommandStatus.FAILED));
182179
Set<Long> failedDocs = failedCommands.stream().map(this::getDocId).collect(Collectors.toSet());
183180

184181
//clean deleted projects that are not actively worked on
185-
Set<Integer> deletedAndNotActive = Sets.difference(toDeleteProjects.keySet(), updatingProjects.keySet());
182+
Set<Integer> deletedAndNotActive = Sets.difference(toDeleteProjects.keySet(), updatingProjects);
186183
for (Integer idx : deletedAndNotActive) {
187184
cleanDeletedProject(toDeleteProjects.get(idx));
188185
}
189186
//determine projects to be excluded
190-
Set<Project> excludeProjects = unionProjects(toDeleteProjects, deletingProjects);
187+
Set<Integer> excludeProjects = unionProjects(toDeleteProjects, deletingProjects);
191188
//determine documents to be excluded
192189
Set<Long> excludeDocs = new HashSet<>();
193190
excludeDocs.addAll(updatingDocs);
@@ -217,10 +214,10 @@ private void processInt() throws CommandException {
217214
}
218215
}
219216

220-
private Set<Project> unionProjects(Map<Integer, SearchFSCommand> p1, Map<Integer, SearchFSCommand> p2) {
221-
Set<Project> result = new HashSet<>();
222-
p1.values().forEach(c -> result.add(c.getProject()));
223-
p2.values().forEach(c -> result.add(c.getProject()));
217+
private Set<Integer> unionProjects(Map<Integer, SearchFSCommand> p1, Map<Integer, SearchFSCommand> p2) {
218+
Set<Integer> result = new HashSet<>();
219+
p1.values().forEach(c -> result.add(c.getProjectId()));
220+
p2.values().forEach(c -> result.add(c.getProjectId()));
224221
return result;
225222
}
226223

@@ -230,20 +227,20 @@ private void cleanDeletedProject(SearchFSCommand command) {
230227
Try<Boolean> result = processFunction().apply(command);
231228
try {
232229
if(result.checkedGet()) {
233-
List<SearchFSCommand> toSkip = commandFacade.findByQuery(queryByProject(command.getProject(),
230+
List<SearchFSCommand> toSkip = commandFacade.findByQuery(queryByProject(command.getProjectId(),
234231
SearchFSCommandOp.DELETE_PROJECT));
235232
toSkip.forEach(c -> removeCommand(c, CommandStatus.SKIPPED));
236233
removeCommand(command, CommandStatus.SUCCESS);
237234
}
238235
} catch (Throwable t) {
239236
LOGGER.log(Level.INFO, "Project:{0} clean failed with error:{1}",
240-
new Object[]{command.getProject().getId(), t.getStackTrace()});
237+
new Object[]{command.getProjectId(), t.getStackTrace()});
241238
failCommand(command, t.getMessage());
242239
}
243240
});
244241
}
245242

246-
private Set<Long> cleanDeletedArtifacts(Set<Project> excludeProjects, Set<Long> excludeDocs, int maxOngoing)
243+
private Set<Long> cleanDeletedArtifacts(Set<Integer> excludeProjects, Set<Long> excludeDocs, int maxOngoing)
247244
throws CommandException {
248245
Set<Long> deleting = new HashSet<>();
249246
List<SearchFSCommand> toDelete = commandFacade
@@ -276,7 +273,7 @@ private void cleanDeletedArtifact(SearchFSCommand command) {
276273
});
277274
}
278275

279-
private Set<Long> cleanDeleteCascadedArtifacts(Set<Project> excludeProjects, Set<Long> excludeDocs, int maxOngoing) {
276+
private Set<Long> cleanDeleteCascadedArtifacts(Set<Integer> excludeProjects, Set<Long> excludeDocs, int maxOngoing) {
280277
Set<Long> deleting = new HashSet<>();
281278
List<SearchFSCommand> toDelete = commandFacade.findDeleteCascaded(excludeProjects, excludeDocs, maxOngoing);
282279
for(SearchFSCommand command : toDelete) {
@@ -298,7 +295,7 @@ private void cleanDeleteCascadedArtifact(SearchFSCommand command) {
298295
cleanDeletedArtifact(deleteArtifact);
299296
}
300297

301-
private Set<Long> processArtifacts(Set<Project> excludeProjects, Set<Long> excludeDocs, int maxOngoing) {
298+
private Set<Long> processArtifacts(Set<Integer> excludeProjects, Set<Long> excludeDocs, int maxOngoing) {
302299
Set<Long> processing = new HashSet<>();
303300
List<SearchFSCommand> toProcess = commandFacade.findToProcess(excludeProjects, excludeDocs, maxOngoing);
304301
for(SearchFSCommand command : toProcess) {
@@ -342,9 +339,9 @@ private QueryParam queryDeletedProjects(CommandStatus status) {
342339
return new QueryParam(null, null, filters, null);
343340
}
344341

345-
private QueryParam queryByProject(Project project, SearchFSCommandOp notOp) {
342+
private QueryParam queryByProject(Integer projectId, SearchFSCommandOp notOp) {
346343
Set<AbstractFacade.FilterBy> filters = new HashSet<>();
347-
filters.add(new CommandFilterBy(CommandFacade.Filters.PROJECT_ID_EQ, project.getId().toString()));
344+
filters.add(new CommandFilterBy(CommandFacade.Filters.PROJECT_ID_EQ, projectId.toString()));
348345
filters.add(new CommandFilterBy(SearchFSCommandFacade.SearchFSFilters.OP_NEQ, notOp.name()));
349346
return new QueryParam(null, null, filters, null);
350347
}
@@ -420,7 +417,7 @@ private Function<SearchFSCommand, Try<Boolean>> processFunction() {
420417
private Try<Boolean> processCommand(SearchFSCommand c) {
421418
try {
422419
if(c.getOp().equals(SearchFSCommandOp.DELETE_PROJECT)) {
423-
searchController.deleteProject(c.getProject());
420+
searchController.deleteProject(c.getProjectId());
424421
return Try.apply(() -> true);
425422
} else if (c.getOp().equals(SearchFSCommandOp.DELETE_ARTIFACT)) {
426423
searchController.delete(c.getInodeId());

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/commands/featurestore/search/SearchFSOpenSearchController.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
import io.hops.hopsworks.persistence.entity.featurestore.featureview.FeatureView;
4444
import io.hops.hopsworks.persistence.entity.featurestore.metadata.FeatureStoreTag;
4545
import io.hops.hopsworks.persistence.entity.featurestore.trainingdataset.TrainingDatasetFeature;
46-
import io.hops.hopsworks.persistence.entity.project.Project;
4746
import io.hops.hopsworks.restutils.RESTCodes;
4847
import org.opensearch.action.delete.DeleteRequest;
4948
import org.opensearch.action.index.IndexRequest;
@@ -86,10 +85,10 @@ public class SearchFSOpenSearchController {
8685
@EJB
8786
private Settings settings;
8887

89-
public long deleteProject(Project project) throws OpenSearchException {
88+
public long deleteProject(Integer projectId) throws OpenSearchException {
9089
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest(Settings.FEATURESTORE_INDEX);
9190
deleteRequest.setQuery(QueryBuilders.matchQuery(
92-
FeaturestoreXAttrsConstants.PROJECT_ID, project.getId()));
91+
FeaturestoreXAttrsConstants.PROJECT_ID, projectId));
9392
return opensearchClient.deleteByQuery(deleteRequest);
9493
}
9594

@@ -153,7 +152,7 @@ private SearchDoc updateMetadata(SearchFSCommand c) throws CommandException {
153152

154153
private SearchDoc create(SearchFSCommand c) throws CommandException {
155154
SearchDoc doc = new SearchDoc();
156-
doc.setProjectId(c.getProject().getId());
155+
doc.setProjectId(c.getProjectId());
157156
doc.setProjectName(c.getProject().getName());
158157

159158
String featureStorePath = Utils.getFeaturestorePath(c.getProject(), settings);

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/commands/CommandFacade.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434

3535
public abstract class CommandFacade<C extends Command> extends AbstractFacade<C> {
3636
public static final String STATUS_FIELD = "status";
37-
public static final String PROJECT_FIELD = "project";
37+
public static final String PROJECT_ID_FIELD = "projectId";
3838

3939
@PersistenceContext(unitName = "kthfsPU")
4040
protected EntityManager em;
@@ -108,7 +108,7 @@ protected void setParam(Query q, AbstractFacade.FilterBy filterBy) throws Comman
108108
try {
109109
if(filterBy.getField().equals(STATUS_FIELD)) {
110110
q.setParameter(filterBy.getField(), CommandStatus.valueOf(filterBy.getParam()));
111-
} else if(filterBy.getField().equals(PROJECT_FIELD)){
111+
} else if(filterBy.getField().equals(PROJECT_ID_FIELD)){
112112
q.setParameter(filterBy.getField(), Integer.parseInt(filterBy.getParam()));
113113
} else {
114114
String msg = "invalid filter:" + filterBy.toString();
@@ -124,8 +124,7 @@ protected void setParam(Query q, AbstractFacade.FilterBy filterBy) throws Comman
124124
public enum Filters implements CommandFilter {
125125
STATUS_EQ(STATUS_FIELD, "c.status = :status ", "NEW"),
126126
STATUS_NEQ(STATUS_FIELD,"c.status != :status ", "NEW"),
127-
PROJECT_ID_EQ(PROJECT_FIELD, "c.project.id = :project", "0"),
128-
PROJECT_IS_NULL(PROJECT_FIELD, "c.project IS NULL", null);
127+
PROJECT_ID_EQ(PROJECT_ID_FIELD, "c." + PROJECT_ID_FIELD + " = :" + PROJECT_ID_FIELD, "0");
129128

130129
private final String sql;
131130
private final String field;

‎hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/commands/search/SearchFSCommandFacade.java‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import io.hops.hopsworks.persistence.entity.commands.CommandStatus;
2424
import io.hops.hopsworks.persistence.entity.commands.search.SearchFSCommand;
2525
import io.hops.hopsworks.persistence.entity.commands.search.SearchFSCommandOp;
26-
import io.hops.hopsworks.persistence.entity.project.Project;
2726
import io.hops.hopsworks.restutils.RESTCodes;
2827

2928
import javax.ejb.Stateless;
@@ -51,15 +50,15 @@ protected String getTableName() {
5150
return SearchFSCommand.TABLE_NAME;
5251
}
5352

54-
public List<SearchFSCommand> findByQuery(QueryParam queryParam, Set<Project> excludeProjects, Set<Long> excludeDocs)
53+
public List<SearchFSCommand> findByQuery(QueryParam queryParam, Set<Integer> excludeProjects, Set<Long> excludeDocs)
5554
throws CommandException {
5655
if(queryParam == null) {
5756
throw new CommandException(RESTCodes.CommandErrorCode.INVALID_SQL_QUERY, Level.INFO, "query param is null");
5857
}
5958
String queryStrPrefix = "SELECT c FROM " + getTableName() + " c ";
6059
String queryStr = buildQuery(queryStrPrefix, queryParam.getFilters(), queryParam.getSorts(), "");
6160
if(!excludeProjects.isEmpty()) {
62-
queryStr += " AND c." + PROJECT_FIELD + " NOT IN :exclude_" + PROJECT_FIELD;
61+
queryStr += " AND c." + PROJECT_ID_FIELD + " NOT IN :exclude_" + PROJECT_ID_FIELD;
6362
}
6463
if(!excludeDocs.isEmpty()) {
6564
queryStr += " AND c." + DOC_ID_FIELD + " NOT IN :exclude_" + DOC_ID_FIELD;
@@ -70,15 +69,15 @@ public List<SearchFSCommand> findByQuery(QueryParam queryParam, Set<Project> exc
7069
q.setMaxResults(queryParam.getLimit());
7170
}
7271
if(!excludeProjects.isEmpty()) {
73-
q.setParameter("exclude_" + PROJECT_FIELD, excludeProjects);
72+
q.setParameter("exclude_" + PROJECT_ID_FIELD, excludeProjects);
7473
}
7574
if(!excludeDocs.isEmpty()) {
7675
q.setParameter("exclude_" + DOC_ID_FIELD, excludeDocs);
7776
}
7877
return q.getResultList();
7978
}
8079

81-
public List<SearchFSCommand> findToProcess(Set<Project> excludeProjects, Set<Long> excludeDocs, int limit) {
80+
public List<SearchFSCommand> findToProcess(Set<Integer> excludeProjects, Set<Long> excludeDocs, int limit) {
8281
UnaryOperator<String> filterFoLive = tableName -> {
8382
String filter = "(";
8483
filter += tableName + "." + FEATURE_GROUP_FIELD + " IS NOT NULL OR ";
@@ -90,7 +89,7 @@ public List<SearchFSCommand> findToProcess(Set<Project> excludeProjects, Set<Lon
9089
return findToProcessInt(excludeProjects, excludeDocs, limit, filterFoLive);
9190
}
9291

93-
public List<SearchFSCommand> findDeleteCascaded(Set<Project> excludeProjects, Set<Long> excludeDocs, int limit) {
92+
public List<SearchFSCommand> findDeleteCascaded(Set<Integer> excludeProjects, Set<Long> excludeDocs, int limit) {
9493
UnaryOperator<String> filterForDeleteCascaded = tableName -> {
9594
String filter = "(";
9695
filter += tableName + "." + FEATURE_GROUP_FIELD + " IS NULL AND ";
@@ -102,7 +101,7 @@ public List<SearchFSCommand> findDeleteCascaded(Set<Project> excludeProjects, Se
102101
return findToProcessInt(excludeProjects, excludeDocs, limit, filterForDeleteCascaded);
103102
}
104103

105-
private List<SearchFSCommand> findToProcessInt(Set<Project> excludeProjects, Set<Long> excludeDocs, int limit,
104+
private List<SearchFSCommand> findToProcessInt(Set<Integer> excludeProjects, Set<Long> excludeDocs, int limit,
106105
UnaryOperator<String> queryAppendFilter) {
107106
String queryStr = "";
108107
queryStr += "SELECT jc FROM " + getTableName() + " jc WHERE jc.id IN (";
@@ -112,7 +111,7 @@ private List<SearchFSCommand> findToProcessInt(Set<Project> excludeProjects, Set
112111
queryStr += " WHERE c.status = :status";
113112
queryStr += " AND " + queryAppendFilter.apply("c");
114113
if(!excludeProjects.isEmpty()) {
115-
queryStr += " AND c." + PROJECT_FIELD + " NOT IN :exclude_" + PROJECT_FIELD;
114+
queryStr += " AND c." + PROJECT_ID_FIELD + " NOT IN :exclude_" + PROJECT_ID_FIELD;
116115
}
117116
if(!excludeDocs.isEmpty()) {
118117
queryStr += " AND c." + DOC_ID_FIELD + " NOT IN :exclude_" + DOC_ID_FIELD;
@@ -124,7 +123,7 @@ private List<SearchFSCommand> findToProcessInt(Set<Project> excludeProjects, Set
124123
TypedQuery<SearchFSCommand> query = em.createQuery(queryStr, entityClass);
125124
query.setParameter("status", CommandStatus.NEW);
126125
if (!excludeProjects.isEmpty()) {
127-
query.setParameter("exclude_" + PROJECT_FIELD, excludeProjects);
126+
query.setParameter("exclude_" + PROJECT_ID_FIELD, excludeProjects);
128127
}
129128
if (!excludeDocs.isEmpty()) {
130129
query.setParameter("exclude_" + DOC_ID_FIELD, excludeDocs);

‎hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/commands/Command.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ public abstract class Command {
4141
@JoinColumn(name = "project_id", referencedColumnName = "id")
4242
@ManyToOne(optional = false)
4343
private Project project;
44+
@Column(name = "project_id", insertable = false, updatable = false)
45+
private Integer projectId;
4446
@Basic(optional = false)
4547
@NotNull
4648
@Size(min = 1, max = 20)
@@ -67,6 +69,10 @@ public Project getProject() {
6769
return project;
6870
}
6971

72+
public Integer getProjectId() {
73+
return projectId;
74+
}
75+
7076
public CommandStatus getStatus() {
7177
return status;
7278
}

‎hopsworks-persistence/src/main/java/io/hops/hopsworks/persistence/entity/commands/search/SearchFSCommandHistory.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ public SearchFSCommandHistory() {
9191

9292
public SearchFSCommandHistory(SearchFSCommand command) {
9393
this.id = command.getId();
94-
this.projectId = command.getProject() != null ? command.getProject().getId() : null;
94+
this.projectId = command.getProjectId();
9595
this.status = command.getStatus();
9696
this.errorMsg = command.getErrorMsg();
9797
this.featureGroupId = command.getFeatureGroup() != null ? command.getFeatureGroup().getId() : null;

0 commit comments

Comments
 (0)