Skip to content

Commit 9131753

Browse files
craigtavernercraigtaverner
authored
Merge pull request #379 from neo4j-contrib/0.28-neo4j-4.2
Initial support for multiple OSM imports
2 parents cb4cb87 + 785554d commit 9131753

File tree

13 files changed

+356
-124
lines changed

13 files changed

+356
-124
lines changed

‎README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,13 @@ This has meant that the spatial library needed a major refactoring to work with
7474
and simply add on the rights to create tokens and indexes. In 0.27.2 we instead use `RestrictedAccessMode`
7575
to restrict the users access right to the built in `AccessModel.Static.SCHEMA` and then boost to enable
7676
index and token writes. The difference is subtle and should only be possible to notice in Enterprise Edition.
77+
* 0.28.0 tackles the ability to import multiple OSM files. The initial solution for Neo4j 4.x made use
78+
of schema indexes keyed by the label and property. However, that means that all OSM imports would share
79+
the same index. If they are completely disjointed data sets, this would not matter. But if you import
80+
overlapping OSM files or different versions of the same file file, a mangled partial merger would result.
81+
0.28.0 solves this by using different indexes, and keeping all imports completely separate.
82+
The more complex problems of importing newer versions, and stitching together overlapping areas, are not
83+
yet solved.
7784

7885
Consequences of the port to Neo4j 4.x:
7986

@@ -347,6 +354,7 @@ The Neo4j Spatial Plugin is available for inclusion in the server version of Neo
347354
* [v0.27.0 for Neo4j 4.0.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.0-neo4j-4.0.3/neo4j-spatial-0.27.0-neo4j-4.0.3-server-plugin.jar?raw=true)
348355
* [v0.27.1 for Neo4j 4.1.7](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.1-neo4j-4.1.7/neo4j-spatial-0.27.1-neo4j-4.1.7-server-plugin.jar?raw=true)
349356
* [v0.27.2 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.27.2-neo4j-4.2.3/neo4j-spatial-0.27.2-neo4j-4.2.3-server-plugin.jar?raw=true)
357+
* [v0.28.0 for Neo4j 4.2.3](https://github.com/neo4j-contrib/m2/blob/master/releases/org/neo4j/neo4j-spatial/0.28.0-neo4j-4.2.3/neo4j-spatial-0.28.0-neo4j-4.2.3-server-plugin.jar?raw=true)
350358

351359
For versions up to 0.15-neo4j-2.3.4:
352360

@@ -463,7 +471,7 @@ Add the following repositories and dependency to your project's pom.xml:
463471
<dependency>
464472
<groupId>org.neo4j</groupId>
465473
<artifactId>neo4j-spatial</artifactId>
466-
<version>0.27.2-neo4j-4.2.3</version>
474+
<version>0.28.0-neo4j-4.2.3</version>
467475
</dependency>
468476
~~~
469477

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
<modelVersion>4.0.0</modelVersion>
2424
<artifactId>neo4j-spatial</artifactId>
2525
<groupId>org.neo4j</groupId>
26-
<version>0.27.2-neo4j-4.2.3</version>
26+
<version>0.28.0-neo4j-4.2.3</version>
2727
<name>Neo4j - Spatial Components</name>
2828
<description>Spatial utilities and components for Neo4j</description>
2929
<url>http://components.neo4j.org/${project.artifactId}/${project.version}</url>

‎src/main/java/org/neo4j/gis/spatial/Constants.java‎

Lines changed: 43 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -19,51 +19,54 @@
1919
*/
2020
package org.neo4j.gis.spatial;
2121

22+
import org.neo4j.graphdb.Label;
23+
2224
/**
2325
* @author Davide Savazzi
2426
*/
2527
public interface Constants {
2628

27-
// Node properties
28-
29-
String PROP_BBOX = "bbox";
30-
String PROP_LAYER = "layer";
31-
String PROP_LAYERNODEEXTRAPROPS = "layerprops";
32-
String PROP_CRS = "layercrs";
33-
String PROP_CREATIONTIME = "ctime";
34-
String PROP_GEOMENCODER = "geomencoder";
35-
String PROP_INDEX_CLASS = "index_class";
36-
String PROP_GEOMENCODER_CONFIG = "geomencoder_config";
37-
String PROP_INDEX_CONFIG = "index_config";
29+
// Node properties
30+
31+
String PROP_BBOX = "bbox";
32+
String PROP_LAYER = "layer";
33+
String PROP_LAYERNODEEXTRAPROPS = "layerprops";
34+
String PROP_CRS = "layercrs";
35+
String PROP_CREATIONTIME = "ctime";
36+
String PROP_GEOMENCODER = "geomencoder";
37+
String PROP_INDEX_CLASS = "index_class";
38+
String PROP_GEOMENCODER_CONFIG = "geomencoder_config";
39+
String PROP_INDEX_CONFIG = "index_config";
3840
String PROP_LAYER_CLASS = "layer_class";
3941

40-
String PROP_TYPE = "gtype";
41-
String PROP_QUERY = "query";
42-
String PROP_WKB = "wkb";
43-
String PROP_WKT = "wkt";
44-
String PROP_GEOM = "geometry";
45-
46-
String[] RESERVED_PROPS = new String[] {
47-
PROP_BBOX,
48-
PROP_LAYER,
49-
PROP_LAYERNODEEXTRAPROPS,
50-
PROP_CRS,
51-
PROP_CREATIONTIME,
52-
PROP_TYPE,
53-
PROP_WKB,
54-
PROP_WKT,
55-
PROP_GEOM
56-
};
57-
58-
59-
// OpenGIS geometry type numbers
60-
61-
int GTYPE_GEOMETRY = 0;
62-
int GTYPE_POINT = 1;
63-
int GTYPE_LINESTRING = 2;
64-
int GTYPE_POLYGON = 3;
65-
int GTYPE_MULTIPOINT = 4;
66-
int GTYPE_MULTILINESTRING = 5;
67-
int GTYPE_MULTIPOLYGON = 6;
68-
42+
String PROP_TYPE = "gtype";
43+
String PROP_QUERY = "query";
44+
String PROP_WKB = "wkb";
45+
String PROP_WKT = "wkt";
46+
String PROP_GEOM = "geometry";
47+
48+
String[] RESERVED_PROPS = new String[]{
49+
PROP_BBOX,
50+
PROP_LAYER,
51+
PROP_LAYERNODEEXTRAPROPS,
52+
PROP_CRS,
53+
PROP_CREATIONTIME,
54+
PROP_TYPE,
55+
PROP_WKB,
56+
PROP_WKT,
57+
PROP_GEOM
58+
};
59+
60+
Label LABEL_LAYER = Label.label("SpatialLayer");
61+
62+
// OpenGIS geometry type numbers
63+
64+
int GTYPE_GEOMETRY = 0;
65+
int GTYPE_POINT = 1;
66+
int GTYPE_LINESTRING = 2;
67+
int GTYPE_POLYGON = 3;
68+
int GTYPE_MULTIPOINT = 4;
69+
int GTYPE_MULTILINESTRING = 5;
70+
int GTYPE_MULTIPOLYGON = 6;
71+
6972
}

‎src/main/java/org/neo4j/gis/spatial/DefaultLayer.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,6 @@ public Node getLayerNode(Transaction tx) {
259259
public void delete(Transaction tx, Listener monitor) {
260260
indexWriter.removeAll(tx, true, monitor);
261261
Node layerNode = getLayerNode(tx);
262-
layerNode.getSingleRelationship(SpatialRelationshipTypes.LAYER, Direction.INCOMING).delete();
263262
layerNode.delete();
264263
layerNodeId = -1L;
265264
}

‎src/main/java/org/neo4j/gis/spatial/SpatialDatabaseService.java‎

Lines changed: 49 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
*/
2020
package org.neo4j.gis.spatial;
2121

22-
import org.locationtech.jts.geom.*;
2322
import org.geotools.referencing.crs.AbstractCRS;
2423
import org.geotools.referencing.crs.DefaultGeographicCRS;
24+
import org.locationtech.jts.geom.*;
2525
import org.neo4j.gis.spatial.encoders.Configurable;
2626
import org.neo4j.gis.spatial.encoders.NativePointEncoder;
2727
import org.neo4j.gis.spatial.encoders.SimplePointEncoder;
@@ -32,7 +32,6 @@
3232
import org.neo4j.gis.spatial.utilities.LayerUtilities;
3333
import org.neo4j.gis.spatial.utilities.ReferenceNodes;
3434
import org.neo4j.graphdb.*;
35-
import org.neo4j.kernel.internal.GraphDatabaseAPI;
3635
import org.opengis.referencing.crs.CoordinateReferenceSystem;
3736

3837
import java.util.ArrayList;
@@ -49,24 +48,52 @@
4948
public class SpatialDatabaseService implements Constants {
5049

5150
public final IndexManager indexManager;
52-
private long spatialRoot = -1;
5351

5452
public SpatialDatabaseService(IndexManager indexManager) {
5553
this.indexManager = indexManager;
5654
}
5755

58-
private Node getSpatialRoot(Transaction tx) {
59-
if (spatialRoot < 0) {
60-
spatialRoot = ReferenceNodes.getReferenceNode(tx, "spatial_root").getId();
56+
public static void assertNotOldModel(Transaction tx) {
57+
Node oldReferenceNode = ReferenceNodes.findDeprecatedReferenceNode(tx, "spatial_root");
58+
if (oldReferenceNode != null) {
59+
throw new IllegalStateException("Old reference node exists - please upgrade the spatial database to the new format");
60+
}
61+
}
62+
63+
public List<String> upgradeFromOldModel(Transaction tx) {
64+
ArrayList<String> layersConverted = new ArrayList<>();
65+
Node oldReferenceNode = ReferenceNodes.findDeprecatedReferenceNode(tx, "spatial_root");
66+
if (oldReferenceNode != null) {
67+
List<Node> layers = new ArrayList<>();
68+
69+
for (Relationship relationship : oldReferenceNode.getRelationships(Direction.OUTGOING, SpatialRelationshipTypes.LAYER)) {
70+
layers.add(relationship.getEndNode());
71+
}
72+
73+
for (Node layer : layers) {
74+
Relationship fromRoot = layer.getSingleRelationship(SpatialRelationshipTypes.LAYER, Direction.INCOMING);
75+
fromRoot.delete();
76+
layer.addLabel(LABEL_LAYER);
77+
layersConverted.add((String) layer.getProperty(PROP_LAYER));
78+
}
79+
80+
if (oldReferenceNode.getRelationships().iterator().hasNext()) {
81+
throw new IllegalStateException("Cannot upgrade - ReferenceNode 'spatial_root' still has relationships other than layers");
82+
}
83+
84+
oldReferenceNode.delete();
6185
}
62-
return tx.getNodeById(spatialRoot);
86+
indexManager.makeIndexFor(tx, "SpatialLayers", LABEL_LAYER, PROP_LAYER);
87+
return layersConverted;
6388
}
6489

6590
public String[] getLayerNames(Transaction tx) {
91+
assertNotOldModel(tx);
6692
List<String> names = new ArrayList<>();
6793

68-
for (Relationship relationship : getSpatialRoot(tx).getRelationships(Direction.OUTGOING, SpatialRelationshipTypes.LAYER)) {
69-
Layer layer = LayerUtilities.makeLayerFromNode(tx, indexManager, relationship.getEndNode());
94+
ResourceIterator<Node> layers = tx.findNodes(LABEL_LAYER);
95+
while (layers.hasNext()) {
96+
Layer layer = LayerUtilities.makeLayerFromNode(tx, indexManager, layers.next());
7097
if (layer instanceof DynamicLayer) {
7198
names.addAll(((DynamicLayer) layer).getLayerNames(tx));
7299
} else {
@@ -78,8 +105,10 @@ public String[] getLayerNames(Transaction tx) {
78105
}
79106

80107
public Layer getLayer(Transaction tx, String name) {
81-
for (Relationship relationship : getSpatialRoot(tx).getRelationships(Direction.OUTGOING, SpatialRelationshipTypes.LAYER)) {
82-
Node node = relationship.getEndNode();
108+
assertNotOldModel(tx);
109+
ResourceIterator<Node> layers = tx.findNodes(LABEL_LAYER);
110+
while (layers.hasNext()) {
111+
Node node = layers.next();
83112
if (name.equals(node.getProperty(PROP_LAYER))) {
84113
return LayerUtilities.makeLayerFromNode(tx, indexManager, node);
85114
}
@@ -88,9 +117,11 @@ public Layer getLayer(Transaction tx, String name) {
88117
}
89118

90119
public Layer getDynamicLayer(Transaction tx, String name) {
120+
assertNotOldModel(tx);
91121
ArrayList<DynamicLayer> dynamicLayers = new ArrayList<>();
92-
for (Relationship relationship : getSpatialRoot(tx).getRelationships(Direction.OUTGOING, SpatialRelationshipTypes.LAYER)) {
93-
Node node = relationship.getEndNode();
122+
ResourceIterator<Node> layers = tx.findNodes(LABEL_LAYER);
123+
while (layers.hasNext()) {
124+
Node node = layers.next();
94125
if (!node.getProperty(PROP_LAYER_CLASS, "").toString().startsWith("DefaultLayer")) {
95126
Layer layer = LayerUtilities.makeLayerFromNode(tx, indexManager, node);
96127
if (layer instanceof DynamicLayer) {
@@ -125,7 +156,7 @@ public DynamicLayer asDynamicLayer(Transaction tx, Layer layer) {
125156
}
126157

127158
public DefaultLayer getOrCreateDefaultLayer(Transaction tx, String name) {
128-
return (DefaultLayer) getOrCreateLayer(tx, name, WKBGeometryEncoder.class, DefaultLayer.class, "");
159+
return (DefaultLayer) getOrCreateLayer(tx, name, WKBGeometryEncoder.class, EditableLayerImpl.class, "");
129160
}
130161

131162
public EditableLayer getOrCreateEditableLayer(Transaction tx, String name, String format, String propertyNameConfig) {
@@ -241,7 +272,7 @@ public Layer createWKBLayer(Transaction tx, String name) {
241272
}
242273

243274
public SimplePointLayer createSimplePointLayer(Transaction tx, String name) {
244-
return createSimplePointLayer(tx, name, null);
275+
return createSimplePointLayer(tx, name, (String[]) null);
245276
}
246277

247278
public SimplePointLayer createSimplePointLayer(Transaction tx, String name, String xProperty, String yProperty) {
@@ -253,7 +284,7 @@ public SimplePointLayer createSimplePointLayer(Transaction tx, String name, Stri
253284
}
254285

255286
public SimplePointLayer createNativePointLayer(Transaction tx, String name) {
256-
return createNativePointLayer(tx, name, null);
287+
return createNativePointLayer(tx, name, (String[]) null);
257288
}
258289

259290
public SimplePointLayer createNativePointLayer(Transaction tx, String name, String locationProperty, String bboxProperty) {
@@ -300,15 +331,13 @@ public Layer createLayer(Transaction tx, String name, Class<? extends GeometryEn
300331
throw new SpatialDatabaseException("Layer " + name + " already exists");
301332

302333
Layer layer = LayerUtilities.makeLayerAndNode(tx, indexManager, name, geometryEncoderClass, layerClass, indexClass);
303-
getSpatialRoot(tx).createRelationshipTo(layer.getLayerNode(tx), SpatialRelationshipTypes.LAYER);
304334
if (encoderConfig != null && encoderConfig.length() > 0) {
305335
GeometryEncoder encoder = layer.getGeometryEncoder();
306336
if (encoder instanceof Configurable) {
307337
((Configurable) encoder).setConfiguration(encoderConfig);
308338
layer.getLayerNode(tx).setProperty(PROP_GEOMENCODER_CONFIG, encoderConfig);
309339
} else {
310-
System.out.println("Warning: encoder configuration '" + encoderConfig
311-
+ "' passed to non-configurable encoder: " + geometryEncoderClass);
340+
System.out.println("Warning: encoder configuration '" + encoderConfig + "' passed to non-configurable encoder: " + geometryEncoderClass);
312341
}
313342
}
314343
if (crs != null && layer instanceof EditableLayer) {
@@ -323,12 +352,10 @@ public void deleteLayer(Transaction tx, String name, Listener monitor) {
323352
layer.delete(tx, monitor);
324353
}
325354

326-
@SuppressWarnings("unchecked")
327355
public static int convertGeometryNameToType(String geometryName) {
328356
if (geometryName == null) return GTYPE_GEOMETRY;
329357
try {
330-
return convertJtsClassToGeometryType((Class<? extends Geometry>) Class.forName("org.locationtech.jts.geom."
331-
+ geometryName));
358+
return convertJtsClassToGeometryType((Class<? extends Geometry>) Class.forName("org.locationtech.jts.geom." + geometryName));
332359
} catch (ClassNotFoundException e) {
333360
System.err.println("Unrecognized geometry '" + geometryName + "': " + e);
334361
return GTYPE_GEOMETRY;

‎src/main/java/org/neo4j/gis/spatial/index/IndexManager.java‎

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,28 @@ public IndexManager(GraphDatabaseAPI db, SecurityContext securityContext) {
4747
this.securityContext = IndexAccessMode.withIndexCreate(securityContext);
4848
}
4949

50+
/**
51+
* Blocking call that spawns a thread to create an index and then waits for that thread to finish.
52+
* This is highly likely to cause deadlocks on index checks, so be careful where it is used.
53+
* Best used if you can commit any other outer transaction first, then run this, and after that
54+
* start a new transaction. For example, see the OSMImport approaching to batching transactions.
55+
* It is possible to use this in procedures with outer transactions if you can ensure the outer
56+
* transactions are read-only.
57+
*/
5058
public IndexDefinition indexFor(Transaction tx, String indexName, Label label, String propertyKey) {
59+
return indexFor(tx, indexName, label, propertyKey, true);
60+
}
61+
62+
/**
63+
* Non-blocking call that spawns a thread to create an index and then waits for that thread to finish.
64+
* Use this especially on indexes that are not immediately needed. Also use it if you have an outer
65+
* transaction that cannot be committed before making this call.
66+
*/
67+
public void makeIndexFor(Transaction tx, String indexName, Label label, String propertyKey) {
68+
indexFor(tx, indexName, label, propertyKey, false);
69+
}
70+
71+
private IndexDefinition indexFor(Transaction tx, String indexName, Label label, String propertyKey, boolean waitFor) {
5172
for (IndexDefinition exists : tx.schema().getIndexes(label)) {
5273
if (exists.getName().equals(indexName)) {
5374
return exists;
@@ -60,15 +81,19 @@ public IndexDefinition indexFor(Transaction tx, String indexName, Label label, S
6081
} else {
6182
IndexMaker indexMaker = new IndexMaker(indexName, label, propertyKey);
6283
Thread indexMakerThread = new Thread(indexMaker, name);
63-
indexMakerThread.start();
64-
try {
65-
indexMakerThread.join();
66-
if (indexMaker.e != null) {
67-
throw new RuntimeException("Failed to make index " + indexMaker.description(), indexMaker.e);
84+
if (waitFor) {
85+
indexMakerThread.start();
86+
try {
87+
indexMakerThread.join();
88+
if (indexMaker.e != null) {
89+
throw new RuntimeException("Failed to make index " + indexMaker.description(), indexMaker.e);
90+
}
91+
return indexMaker.index;
92+
} catch (InterruptedException e) {
93+
throw new RuntimeException("Failed to make index " + indexMaker.description(), e);
6894
}
69-
return indexMaker.index;
70-
} catch (InterruptedException e) {
71-
throw new RuntimeException("Failed to make index " + indexMaker.description(), e);
95+
} else {
96+
return null;
7297
}
7398
}
7499
}

0 commit comments

Comments
 (0)