diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloBulkIngester.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloBulkIngester.java index 0d8b5a1..c719b71 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloBulkIngester.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloBulkIngester.java @@ -32,6 +32,12 @@ import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.GraphFactory; import com.tinkerpop.blueprints.Vertex; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.mutator.edge.EdgeEndpointsMutator; +import edu.jhuapl.tinkerpop.mutator.edge.EdgeMutator; +import edu.jhuapl.tinkerpop.mutator.property.WritePropertyMutator; +import edu.jhuapl.tinkerpop.mutator.vertex.AddVertexMutator; + /** * This class provides high-speed ingest into an {@link AccumuloGraph} instance @@ -91,8 +97,8 @@ public final class AccumuloBulkIngester { AccumuloGraphUtils.handleCreateAndClear(config); mtbw = connector.createMultiTableBatchWriter(config.getBatchWriterConfig()); - vertexWriter = mtbw.getBatchWriter(config.getVertexTable()); - edgeWriter = mtbw.getBatchWriter(config.getEdgeTable()); + vertexWriter = mtbw.getBatchWriter(config.getVertexTableName()); + edgeWriter = mtbw.getBatchWriter(config.getEdgeTableName()); } /** @@ -108,9 +114,7 @@ public final class AccumuloBulkIngester { * @throws MutationsRejectedException */ public PropertyBuilder addVertex(String id) throws MutationsRejectedException { - Mutation m = new Mutation(id); - m.put(AccumuloGraph.LABEL, AccumuloGraph.EXISTS, AccumuloGraph.EMPTY); - vertexWriter.addMutation(m); + Mutators.apply(vertexWriter, new AddVertexMutator(id)); return new PropertyBuilder(vertexWriter, id); } @@ -148,8 +152,7 @@ public final class AccumuloBulkIngester { * @throws MutationsRejectedException */ public PropertyBuilder addEdge(String src, String dest, String label) throws MutationsRejectedException { - String eid = UUID.randomUUID().toString(); - return addEdge(eid, src, dest, label); + return addEdge(UUID.randomUUID().toString(), src, dest, label); } /** @@ -168,16 +171,8 @@ public final class AccumuloBulkIngester { * @throws MutationsRejectedException */ public PropertyBuilder addEdge(String id, String src, String dest, String label) throws MutationsRejectedException { - Mutation m = new Mutation(id); - m.put(AccumuloGraph.LABEL, (dest + "_" + src).getBytes(), AccumuloByteSerializer.serialize(label)); - edgeWriter.addMutation(m); - - m = new Mutation(dest); - m.put(AccumuloGraph.INEDGE, (src + AccumuloGraph.IDDELIM + id).getBytes(), (AccumuloGraph.IDDELIM + label).getBytes()); - vertexWriter.addMutation(m); - m = new Mutation(src); - m.put(AccumuloGraph.OUTEDGE, (dest + AccumuloGraph.IDDELIM + id).getBytes(), (AccumuloGraph.IDDELIM + label).getBytes()); - vertexWriter.addMutation(m); + Mutators.apply(edgeWriter, new EdgeMutator.Add(id, src, dest, label)); + Mutators.apply(vertexWriter, new EdgeEndpointsMutator.Add(id, src, dest, label)); return new PropertyBuilder(edgeWriter, id); } @@ -210,10 +205,7 @@ public final class AccumuloBulkIngester { * @throws MutationsRejectedException */ private void addProperty(BatchWriter writer, String id, String key, Object value) throws MutationsRejectedException { - byte[] newByteVal = AccumuloByteSerializer.serialize(value); - Mutation m = new Mutation(id); - m.put(key.getBytes(), AccumuloGraph.EMPTY, newByteVal); - writer.addMutation(m); + Mutators.apply(writer, new WritePropertyMutator(id, key, value)); } /** @@ -279,12 +271,12 @@ public final class AccumuloBulkIngester { */ public final class PropertyBuilder { - Mutation mutation; - BatchWriter writer; + final String id; + final BatchWriter writer; PropertyBuilder(BatchWriter writer, String id) { this.writer = writer; - this.mutation = new Mutation(id); + this.id = id; } /** @@ -296,7 +288,13 @@ public final class AccumuloBulkIngester { * @return */ public PropertyBuilder add(String key, Object value) { - mutation.put(key.getBytes(), AccumuloGraph.EMPTY, AccumuloByteSerializer.serialize(value)); + for (Mutation m : new WritePropertyMutator(id, key, value).create()) { + try { + writer.addMutation(m); + } catch (MutationsRejectedException e) { + throw new AccumuloGraphException(e); + } + } return this; } @@ -306,9 +304,7 @@ public final class AccumuloBulkIngester { * @throws MutationsRejectedException */ public void finish() throws MutationsRejectedException { - if (mutation.size() > 0) { - writer.addMutation(mutation); - } + // No-op since Mutations are now added on the fly. } /** @@ -317,7 +313,7 @@ public final class AccumuloBulkIngester { * @return */ public String getId() { - return new String(mutation.getRow()); + return id; } } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloByteSerializer.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloByteSerializer.java index b8065c1..74b85d3 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloByteSerializer.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloByteSerializer.java @@ -26,21 +26,21 @@ import javax.xml.namespace.QName; public final class AccumuloByteSerializer { - static final int NULL = 'n'; + public static final int NULL = 'n'; - static final int BYTE = 'b'; - static final int SHORT = 's'; - static final int CHARACTER = 'c'; - static final int INTEGER = 'i'; - static final int LONG = 'l'; - static final int FLOAT = 'f'; - static final int DOUBLE = 'd'; - static final int BOOLEAN = 'o'; - static final int DATE = 't'; - static final int ENUM = 'e'; - static final int STRING = 'a'; - static final int SERIALIZABLE = 'x'; - static final int QNAME = 'q'; + public static final int BYTE = 'b'; + public static final int SHORT = 's'; + public static final int CHARACTER = 'c'; + public static final int INTEGER = 'i'; + public static final int LONG = 'l'; + public static final int FLOAT = 'f'; + public static final int DOUBLE = 'd'; + public static final int BOOLEAN = 'o'; + public static final int DATE = 't'; + public static final int ENUM = 'e'; + public static final int STRING = 'a'; + public static final int SERIALIZABLE = 'x'; + public static final int QNAME = 'q'; private AccumuloByteSerializer() { @@ -53,6 +53,7 @@ public final class AccumuloByteSerializer { } }; + @SuppressWarnings("unchecked") public static T deserialize(byte[] target) { if (target[0] == NULL) { return null; @@ -92,6 +93,7 @@ public final class AccumuloByteSerializer { case ENUM: try { String[] s = new String(target, 1, target.length - 1).split(":"); + @SuppressWarnings("rawtypes") Class clz = (Class) Class.forName(s[0]); return (T) Enum.valueOf(clz, s[1]); } catch (ClassNotFoundException cnfe) { diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloEdge.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloEdge.java index 3fe7893..94f011b 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloEdge.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloEdge.java @@ -14,70 +14,50 @@ */ package edu.jhuapl.tinkerpop; +import java.util.Map; +import org.apache.log4j.Logger; + import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; -import com.tinkerpop.blueprints.util.ExceptionFactory; import com.tinkerpop.blueprints.util.StringFactory; +/** + * TODO + */ public class AccumuloEdge extends AccumuloElement implements Edge { - String label; - String inId; - String outId; - Vertex inVertex; - Vertex outVertex; + private static final Logger log = Logger.getLogger(AccumuloEdge.class); - AccumuloEdge(AccumuloGraph parent, String id) { - this(parent, id, null); + private String label; + private Vertex inVertex; + private Vertex outVertex; + + public AccumuloEdge(GlobalInstances globals, String id) { + this(globals, id, null, null, null); } - AccumuloEdge(AccumuloGraph parent, String id, String label) { - this(parent, id, label, (Vertex) null, (Vertex) null); - } - - AccumuloEdge(AccumuloGraph parent, String id, String label, Vertex inVertex, Vertex outVertex) { - super(parent, id, Edge.class); + public AccumuloEdge(GlobalInstances globals, String id, + Vertex inVertex, Vertex outVertex, String label) { + super(globals, id, Edge.class); this.label = label; this.inVertex = inVertex; this.outVertex = outVertex; } - AccumuloEdge(AccumuloGraph parent, String id, String label, String inVertex, String outVertex) { - super(parent, id, Edge.class); - this.label = label; - this.inId = inVertex; - this.outId = outVertex; - } - @Override public Vertex getVertex(Direction direction) throws IllegalArgumentException { - switch (direction) { - case IN: - if (inVertex == null) { - if (inId == null) { - inVertex = parent.getEdgeVertex(id, direction); - inId = inVertex.getId().toString(); - } else { - inVertex = parent.getVertex(inId); - } - } - return inVertex; - case OUT: - if (outVertex == null) { - if (outId == null) { - outVertex = parent.getEdgeVertex(id, direction); - outId = outVertex.getId().toString(); - } else { - outVertex = parent.getVertex(outId); - } - } - return outVertex; - case BOTH: - throw ExceptionFactory.bothIsNotSupported(); - default: - throw new RuntimeException("Unexpected direction: " + direction); + if (!Direction.IN.equals(direction) && !Direction.OUT.equals(direction)) { + throw new IllegalArgumentException("Invalid direction: "+direction); } + + // The vertex information needs to be loaded. + if (inVertex == null || outVertex == null || label == null) { + log.debug("Loading information for edge: "+this); + globals.getEdgeWrapper().loadEndpointsAndLabel(this); + } + + return Direction.IN.equals(direction) ? inVertex : outVertex; } @Override @@ -91,31 +71,45 @@ public class AccumuloEdge extends AccumuloElement implements Edge { @Override public void remove() { - parent.removeEdge(this); + // Remove from named indexes. + super.removeElementFromNamedIndexes(); + + // If edge was removed already, forget it. + // This may happen due to self-loops... + if (!globals.getEdgeWrapper().elementExists(id)) { + return; + } + + // Remove properties from key/value indexes. + Map props = globals.getEdgeWrapper() + .readAllProperties(this); + + for (String key : props.keySet()) { + globals.getEdgeKeyIndexWrapper().removePropertyFromIndex(this, + key, props.get(key)); + } + + // Get rid of the endpoints and edge themselves. + globals.getVertexWrapper().deleteEdgeEndpoints(this); + globals.getEdgeWrapper().deleteEdge(this); + + // Remove element from cache. + globals.getCaches().remove(id, Edge.class); + + globals.checkedFlush(); } - public String getInId() { - return inId; + public void setVertices(AccumuloVertex inVertex, AccumuloVertex outVertex) { + this.inVertex = inVertex; + this.outVertex = outVertex; } - public String getOutId() { - return outId; - } - - protected void setInId(String id){ - inId = id; - } - - protected void setOutId(String id){ - outId = id; - } - - protected void setLabel(String label){ + public void setLabel(String label) { this.label = label; } @Override public String toString() { - return "[" + getId() + ":" + getVertex(Direction.OUT) + " -> " + getLabel() + " -> " + getVertex(Direction.IN) + "]"; + return "[" + getId() + ":" + inVertex + " -> " + label + " -> " + outVertex + "]"; } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloElement.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloElement.java index 91f9834..faf7cd8 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloElement.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloElement.java @@ -16,71 +16,153 @@ package edu.jhuapl.tinkerpop; import java.util.Set; -import org.apache.accumulo.core.util.Pair; - import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Index; +import com.tinkerpop.blueprints.util.StringFactory; +import edu.jhuapl.tinkerpop.cache.PropertyCache; + +/** + * TODO + */ public abstract class AccumuloElement implements Element { - protected AccumuloGraph parent; + protected GlobalInstances globals; protected String id; private Class type; private PropertyCache propertyCache; - protected AccumuloElement(AccumuloGraph parent, String id, Class type) { - this.parent = parent; + protected AccumuloElement(GlobalInstances globals, + String id, Class type) { + this.globals = globals; this.id = id; this.type = type; } - @Override - public T getProperty(String key) { + /** + * Create properties cache if it doesn't exist, + * and preload any properties. + */ + private void makeCache() { if (propertyCache == null) { - // Lazily create the properties cache. - // We will create it here just in case the parent does not actually - // pre-load any data. Note it also may be created in the - // cacheProperty method, as well, in the event a class pre-loads - // data before a call is made to obtain it. - propertyCache = new PropertyCache(parent.config); + propertyCache = new PropertyCache(globals.getConfig()); - parent.preloadProperties(this, type); - } - - T val = propertyCache.get(key); - if (val != null) { - return val; - } else { - Pair pair = parent.getProperty(type, id, key); - if (pair.getFirst() != null) { - cacheProperty(key, pair.getSecond()); + // Preload any keys, if needed. + String[] preloadKeys = globals.getConfig().getPreloadedProperties(); + if (preloadKeys != null) { + propertyCache.putAll(globals.getElementWrapper(type) + .readProperties(this, preloadKeys)); } - return pair.getSecond(); } } + @Override + public T getProperty(String key) { + makeCache(); + + // Get from property cache. + T value = propertyCache.get(key); + + // If not cached, get it from the backing table. + if (value == null) { + value = globals.getElementWrapper(type).readProperty(this, key); + } + + // Cache the new value. + if (value != null) { + propertyCache.put(key, value); + } + + return value; + } + + @Override public Set getPropertyKeys() { - return parent.getPropertyKeys(type, id); + return globals.getElementWrapper(type).readPropertyKeys(this); } @Override public void setProperty(String key, Object value) { - parent.setProperty(type, id, key, value); - cacheProperty(key, value); + makeCache(); + globals.getKeyIndexTableWrapper(type).setPropertyForIndex(this, key, value); + // MDL 31 Dec 2014: The above calls getProperty, so this + // order is important (for now). + globals.getElementWrapper(type).writeProperty(this, key, value); + globals.checkedFlush(); + setPropertyInMemory(key, value); + } + + /** + * Set a property but only in the instantiated object, + * not in the backing store. + * @param key + * @param value + */ + public void setPropertyInMemory(String key, Object value) { + makeCache(); + propertyCache.put(key, value); } @Override public T removeProperty(String key) { - if (propertyCache != null) { - // we have the cached value but we still need to pass this on to the - // parent so it can actually remove the data from the backing store. - // Since we have to do that anyway, we will use the parent's value - // instead of the cache value to to be as up-to-date as possible. - // Of course we still need to clear out the cached value... - propertyCache.remove(key); + if (StringFactory.LABEL.equals(key) || + Constants.LABEL.equals(key)) { + throw new AccumuloGraphException("Cannot remove the " + StringFactory.LABEL + " property."); } - return parent.removeProperty(type, id, key); + + makeCache(); + T value = getProperty(key); + if (value != null) { + globals.getElementWrapper(type).clearProperty(this, key); + globals.checkedFlush(); + } + globals.getKeyIndexTableWrapper(type).removePropertyFromIndex(this, key, value); + // MDL 31 Dec 2014: AccumuloGraph.removeProperty + // calls getProperty which populates the cache. + // So the order here is important (for now). + removePropertyInMemory(key); + return value; + } + + /** + * Remove element from all named indexes. + * @param element + */ + protected void removeElementFromNamedIndexes() { + for (Index index : globals.getNamedIndexListWrapper().getIndices()) { + ((AccumuloIndex) index).getWrapper().removeElementFromIndex(this); + } + } + + /** + * Remove a property but only in the instantiated + * object, not the backing store. + * @param key + */ + public void removePropertyInMemory(String key) { + makeCache(); + propertyCache.remove(key); + } + + /** + * Return the properties currently cached in memory. + * @return + */ + public Iterable getPropertyKeysInMemory() { + makeCache(); + return propertyCache.keySet(); + } + + /** + * Retrieve a property from memory. + * @param key + * @return + */ + public Object getPropertyInMemory(String key) { + makeCache(); + return propertyCache.get(key); } @Override @@ -97,19 +179,20 @@ public abstract class AccumuloElement implements Element { } else if (!obj.getClass().equals(getClass())) { return false; } else { - return this.id.equals(((AccumuloElement) obj).id); + return id.equals(((AccumuloElement) obj).id); } } + @Override public int hashCode() { return getClass().hashCode() ^ id.hashCode(); } - void cacheProperty(String key, Object value) { - if (propertyCache == null) { - propertyCache = new PropertyCache(parent.config); - } - propertyCache.put(key, value); + /** + * Internal method for unit tests. + * @return + */ + PropertyCache getPropertyCache() { + return propertyCache; } - } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraph.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraph.java index 59abaf8..751862c 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraph.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraph.java @@ -14,44 +14,24 @@ */ package edu.jhuapl.tinkerpop; -import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; +import java.util.Map; import java.util.Set; import java.util.SortedSet; -import java.util.UUID; -import java.util.regex.Pattern; import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.BatchDeleter; -import org.apache.accumulo.core.client.BatchScanner; -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.IteratorSetting; -import org.apache.accumulo.core.client.MultiTableBatchWriter; import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.iterators.user.RegExFilter; -import org.apache.accumulo.core.util.Pair; -import org.apache.accumulo.core.util.PeekingIterator; import org.apache.commons.configuration.Configuration; import org.apache.hadoop.io.Text; -import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Element; import com.tinkerpop.blueprints.Features; import com.tinkerpop.blueprints.Graph; +import com.tinkerpop.blueprints.GraphFactory; import com.tinkerpop.blueprints.GraphQuery; import com.tinkerpop.blueprints.Index; import com.tinkerpop.blueprints.IndexableGraph; @@ -60,229 +40,71 @@ import com.tinkerpop.blueprints.Parameter; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.util.DefaultGraphQuery; import com.tinkerpop.blueprints.util.ExceptionFactory; -import com.tinkerpop.blueprints.util.StringFactory; + +import edu.jhuapl.tinkerpop.cache.ElementCaches; /** + * This is an implementation of the TinkerPop Blueprints 2.6 API using + * Apache Accumulo as the backend. This combines the many benefits and flexibility + * of Blueprints with the scalability and performance of Accumulo. * - * This is an implementation of TinkerPop's graph API - * backed by Apache Accumulo. In addition to the basic - * Graph interface, the implementation - * supports {@link IndexableGraph} and {@link KeyIndexableGraph}. - * - *

Tables have the following formats. - * - *

- * - * - * - * - * - * - * - * - * - * - * - *
Vertex table
ROWIDCOLFAMCOLQUALVALUE
VertexIDLABELEXISTS[empty]
VertexIDINEDGEInVertexID_EdgeIDEdgeLabel
VertexIDOUTEDGEOutVertexID_EdgeIDEdgeLabel
VertexIDPropertyKey[empty]Encoded Value
- * - *

- * - * - * - * - * - * - * - * - * - * - * - *
Edge table
ROWIDCOLFAMCOLQUALVALUE
EdgeIDLABEL[empty]Encoded LabelValue
EdgeIDINEDGEInVertexID[empty]
EdgeIDOUTEDGEOutVertexID[empty]
EdgeIDPropertyKey[empty]Encoded Value
- * - *

- * - * - * - * - * - * - * - * - *
Vertex / edge index tables (each index gets its own table)
ROWIDCOLFAMCOLQUALVALUE
Encoded ValuePropertyKeyElementID[empty]
- * - *

- * - * - * - * - * - * - * - * - *
Metadata/key metadata tables
ROWIDCOLFAMCOLQUALVALUE
IndexNameIndexClassType[empty][empty]
+ *

In addition to the basic Blueprints functionality, we provide a number of + * enhanced features, including: + *

    + *
  1. Indexing implementations via IndexableGraph and KeyIndexableGraph
    2. + *
  2. Support for mock, mini, and distributed instances of Accumulo
    3. + *
  3. Numerous performance tweaks and configuration parameters
    4. + *
  4. Support for high speed ingest
    5. + *
  5. Hadoop integration
    6. + *
*/ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { - protected AccumuloGraphConfiguration config; - - static byte[] EMPTY = new byte[0]; - - public static final String IDDELIM = "_"; - public static final String SLABEL = "L"; - public static final String SINEDGE = "I"; - public static final String SOUTEDGE = "O"; - public static final String SEXISTS = "E"; - public static final byte[] EXISTS = SEXISTS.getBytes(); - public static final byte[] LABEL = SLABEL.getBytes(); - public static final byte[] INEDGE = SINEDGE.getBytes(); - public static final byte[] OUTEDGE = SOUTEDGE.getBytes(); - protected static final Text TEXISTS = new Text(EXISTS); - protected static final Text TINEDGE = new Text(INEDGE); - protected static final Text TOUTEDGE = new Text(OUTEDGE); - protected static final Text TLABEL = new Text(LABEL); - - MultiTableBatchWriter writer; - BatchWriter vertexBW; - BatchWriter edgeBW; - - ElementCache vertexCache; - ElementCache edgeCache; - - public AccumuloGraph(Configuration cfg) { - this(new AccumuloGraphConfiguration(cfg)); - } + private final GlobalInstances globals; /** - * Constructor that ensures that the needed tables are made - * - * @param config - */ - public AccumuloGraph(AccumuloGraphConfiguration config) { - config.validate(); - this.config = config; - - if (config.getVertexCacheEnabled()) { - vertexCache = new ElementCache(config.getVertexCacheSize(), - config.getVertexCacheTimeout()); - } - - if (config.getEdgeCacheEnabled()) { - edgeCache = new ElementCache(config.getEdgeCacheSize(), - config.getEdgeCacheTimeout()); - } - - AccumuloGraphUtils.handleCreateAndClear(config); - try { - setupWriters(); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } - } - - private void setupWriters() throws Exception { - writer = config.getConnector().createMultiTableBatchWriter(config.getBatchWriterConfig()); - - vertexBW = writer.getBatchWriter(config.getVertexTable()); - edgeBW = writer.getBatchWriter(config.getEdgeTable()); - } - - /** - * Factory method for GraphFactory + * Factory method for {@link GraphFactory}. */ public static AccumuloGraph open(Configuration properties) throws AccumuloException { return new AccumuloGraph(properties); } - protected Scanner getElementScanner(Class type) { + /** + * Instantiate from a generic {@link Configuration} populated + * with appropriate AccumuloGraph parameters. + * @param cfg + */ + public AccumuloGraph(Configuration cfg) { + this(new AccumuloGraphConfiguration(cfg)); + } + + /** + * Main constructor. + * @param config + */ + public AccumuloGraph(AccumuloGraphConfiguration config) { + config.validate(); + + AccumuloGraphUtils.handleCreateAndClear(config); + try { - String tableName = config.getEdgeTable(); - if (type.equals(Vertex.class)) - tableName = config.getVertexTable(); - return config.getConnector().createScanner(tableName, config.getAuthorizations()); + globals = new GlobalInstances(config, config.getConnector() + .createMultiTableBatchWriter(config.getBatchWriterConfig()), + new ElementCaches(config)); } catch (Exception e) { throw new AccumuloGraphException(e); } } - protected Scanner getScanner(String tablename) { - try { - return config.getConnector().createScanner(tablename, config.getAuthorizations()); - } catch (TableNotFoundException e) { - e.printStackTrace(); - } catch (AccumuloException e) { - e.printStackTrace(); - } catch (AccumuloSecurityException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return null; + /** + * This is at package level for the {@link AutoIndexTest} unit test + * and probably should disappear. + * @return + */ + GlobalInstances getGlobals() { + return globals; } - // Aliases for the lazy - protected Scanner getMetadataScanner() { - return getScanner(config.getMetadataTable()); - } - - protected Scanner getVertexIndexScanner() { - return getScanner(config.getVertexIndexTable()); - } - - protected Scanner getEdgeIndexScanner() { - return getScanner(config.getEdgeIndexTable()); - } - - protected BatchWriter getVertexIndexWriter() { - return getWriter(config.getVertexIndexTable()); - } - - protected BatchWriter getMetadataWriter() { - return getWriter(config.getMetadataTable()); - } - - protected BatchWriter getEdgeIndexWriter() { - return getWriter(config.getEdgeIndexTable()); - } - - private Scanner getKeyMetadataScanner() { - - return getScanner(config.getMetadataTable() + "KEY"); - } - - protected BatchWriter getKeyMetadataWriter() { - return getWriter(config.getMetadataTable() + "KEY"); - } - - public BatchWriter getWriter(String tablename) { - try { - return writer.getBatchWriter(tablename); - } catch (AccumuloException e) { - e.printStackTrace(); - } catch (AccumuloSecurityException e) { - e.printStackTrace(); - } catch (TableNotFoundException e) { - e.printStackTrace(); - } - return null; - } - - private BatchScanner getElementBatchScanner(Class type) { - try { - String tableName = config.getVertexTable(); - if (type.equals(Edge.class)) - tableName = config.getEdgeTable(); - BatchScanner x = config.getConnector().createBatchScanner(tableName, config.getAuthorizations(), config.getQueryThreads()); - x.setRanges(Collections.singletonList(new Range())); - return x; - } catch (Exception e) { - throw new AccumuloGraphException(e); - } - } - - // End Aliases - @Override public Features getFeatures() { return AccumuloFeatures.get(); @@ -290,41 +112,27 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { @Override public Vertex addVertex(Object id) { - String myID; if (id == null) { - myID = UUID.randomUUID().toString(); - } else { - try { - myID = id.toString();// (String) id; - } catch (ClassCastException e) { - return null; - } + id = AccumuloGraphUtils.generateId(); } + String idStr = id.toString(); + Vertex vert = null; - if (!config.getSkipExistenceChecks()) { - vert = getVertex(myID); + if (!globals.getConfig().getSkipExistenceChecks()) { + vert = getVertex(idStr); if (vert != null) { - throw ExceptionFactory.vertexWithIdAlreadyExists(myID); + throw ExceptionFactory.vertexWithIdAlreadyExists(idStr); } } - Mutation m = new Mutation(myID); - m.put(LABEL, EXISTS, EMPTY); + vert = new AccumuloVertex(globals, idStr); - try { - vertexBW.addMutation(m); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - return null; - } + globals.getVertexWrapper().writeVertex(vert); + globals.checkedFlush(); - checkedFlush(); - vert = new AccumuloVertex(this, myID); + globals.getCaches().cache(vert, Vertex.class); - if (vertexCache != null) { - vertexCache.cache(vert); - } return vert; } @@ -335,306 +143,59 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { } String myID = id.toString(); - if (vertexCache != null) { - Vertex vertex = vertexCache.retrieve(myID); - if (vertex != null) { - return vertex; + Vertex vertex = globals.getCaches().retrieve(myID, Vertex.class); + if (vertex != null) { + return vertex; + } + + vertex = new AccumuloVertex(globals, myID); + if (!globals.getConfig().getSkipExistenceChecks()) { + // In addition to just an "existence" check, we will also load + // any "preloaded" properties now, which saves us a round-trip + // to Accumulo later. + String[] preload = globals.getConfig().getPreloadedProperties(); + if (preload == null) { + preload = new String[]{}; + } + + Map props = globals.getVertexWrapper().readProperties(vertex, preload); + if (props == null) { + return null; + } + + for (String key : props.keySet()) { + ((AccumuloElement) vertex).setPropertyInMemory(key, props.get(key)); } } - Vertex vertex = new AccumuloVertex(this, myID); + globals.getCaches().cache(vertex, Vertex.class); - Scanner scan = null; - try { - if (!config.getSkipExistenceChecks()) { - // in addition to just an "existence" check, we will also load - // any "preloaded" properties now, which saves us a round-trip - // to Accumulo later. - scan = getElementScanner(Vertex.class); - scan.setRange(new Range(myID)); - scan.fetchColumn(TLABEL, TEXISTS); - - String[] preload = config.getPreloadedProperties(); - if (preload != null) { - // user has requested specific properties. - Text colf = new Text(""); - for (String key : preload) { - if (StringFactory.LABEL.equals(key)) { - colf.set(AccumuloGraph.LABEL); - } else { - colf.set(key); - } - scan.fetchColumnFamily(colf); - } - } - - Iterator> iter = scan.iterator(); - if (!iter.hasNext()) { - return null; - } - - preloadProperties(iter, (AccumuloElement) vertex); - - } - } finally { - if (scan != null) { - scan.close(); - } - } - - if (vertexCache != null) { - vertexCache.cache(vertex); - } return vertex; } @Override public void removeVertex(Vertex vertex) { - if (vertexCache != null) { - vertexCache.remove(vertex.getId()); - } - if (!config.getIndexableGraphDisabled()) - clearIndex(vertex.getId()); - - Scanner scan = getElementScanner(Vertex.class); - scan.setRange(new Range(vertex.getId().toString())); - - BatchDeleter edgedeleter = null; - BatchDeleter vertexdeleter = null; - BatchWriter indexdeleter = getVertexIndexWriter(); - try { - // Set up Deleters - edgedeleter = config.getConnector().createBatchDeleter(config.getEdgeTable(), config.getAuthorizations(), config.getQueryThreads(), - config.getBatchWriterConfig()); - vertexdeleter = config.getConnector().createBatchDeleter(config.getVertexTable(), config.getAuthorizations(), config.getQueryThreads(), - config.getBatchWriterConfig()); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } - Iterator> iter = scan.iterator(); - List ranges = new ArrayList(); - if (!iter.hasNext()) { - throw ExceptionFactory.vertexWithIdDoesNotExist(vertex.getId()); - } - try { - // Search for edges - while (iter.hasNext()) { - Entry e = iter.next(); - Key k = e.getKey(); - - if (k.getColumnFamily().equals(TOUTEDGE) || k.getColumnFamily().equals(TINEDGE)) { - ranges.add(new Range(k.getColumnQualifier().toString().split(IDDELIM)[1])); - - Mutation vm = new Mutation(k.getColumnQualifier().toString().split(IDDELIM)[0]); - vm.putDelete(invert(k.getColumnFamily()), new Text(vertex.getId().toString() + IDDELIM + k.getColumnQualifier().toString().split(IDDELIM)[1])); - vertexBW.addMutation(vm); - } else { - Mutation m = new Mutation(e.getValue().get()); - m.putDelete(k.getColumnFamily(), k.getRow()); - indexdeleter.addMutation(m); - } - - } - checkedFlush(); - scan.close(); - - // If Edges are found, delete the whole row - if (!ranges.isEmpty()) { - // TODO don't we also have to propagate these deletes to the - // vertex index table? - edgedeleter.setRanges(ranges); - edgedeleter.delete(); - ranges.clear(); - } - // Delete the whole vertex row - ranges.add(new Range(vertex.getId().toString())); - vertexdeleter.setRanges(ranges); - vertexdeleter.delete(); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } finally { - if (edgedeleter != null) - edgedeleter.close(); - if (vertexdeleter != null) - vertexdeleter.close(); - } - } - - // Maybe an Custom Iterator could make this better. - private void clearIndex(Object id) { - Iterable> it = this.getIndices(); - Iterator> iter = it.iterator(); - while (iter.hasNext()) { - AccumuloIndex in = (AccumuloIndex) iter.next(); - String table = in.tableName; - - BatchDeleter del = null; - try { - del = config.getConnector().createBatchDeleter(table, config.getAuthorizations(), config.getMaxWriteThreads(), config.getBatchWriterConfig()); - del.setRanges(Collections.singleton(new Range())); - StringBuilder regex = new StringBuilder(); - regex.append(".*\\Q").append(id.toString()).append("\\E$"); - - IteratorSetting is = new IteratorSetting(10, "getEdgeFilter", RegExFilter.class); - RegExFilter.setRegexs(is, null, null, regex.toString(), null, false); - del.addScanIterator(is); - del.delete(); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } finally { - if (del != null) - del.close(); - } - } - } - - private Text invert(Text columnFamily) { - if (columnFamily.toString().equals(INEDGE)) { - return TOUTEDGE; - } - return TINEDGE; + vertex.remove(); } @Override public Iterable getVertices() { - Scanner scan = getElementScanner(Vertex.class); - scan.fetchColumnFamily(TLABEL); - - if (config.getPreloadedProperties() != null) { - for (String x : config.getPreloadedProperties()) { - scan.fetchColumnFamily(new Text(x)); - } - } - return new ScannerIterable(this, scan) { - - @Override - public Vertex next(PeekingIterator> iterator) { - // TODO could also check local cache before creating a new instance? - AccumuloVertex vert = new AccumuloVertex(AccumuloGraph.this, iterator.peek().getKey().getRow().toString()); - - String rowid = iterator.next().getKey().getRow().toString(); - List> vals = new ArrayList>(); - while (iterator.peek() != null && rowid.compareToIgnoreCase(iterator.peek().getKey().getRow().toString()) == 0) { - vals.add(iterator.next()); - } - preloadProperties(vals.iterator(), vert); - return vert; - } - }; + return globals.getVertexWrapper().getVertices(); } @Override public Iterable getVertices(String key, Object value) { - checkProperty(key, value); - if (config.getAutoIndex() || getIndexedKeys(Vertex.class).contains(key)) { - // Use the index - Scanner s = getVertexIndexScanner(); - byte[] val = AccumuloByteSerializer.serialize(value); - Text tVal = new Text(val); - s.setRange(new Range(tVal, tVal)); - s.fetchColumnFamily(new Text(key)); - - return new ScannerIterable(this, s) { - - @Override - public Vertex next(PeekingIterator> iterator) { - - Key key = iterator.next().getKey(); - AccumuloVertex v = null; - if (vertexCache != null) { - v = (AccumuloVertex) vertexCache.retrieve(key.getColumnQualifier().toString()); - } - - v = (v == null ? new AccumuloVertex(AccumuloGraph.this, key.getColumnQualifier().toString()) : v); - v.cacheProperty(key.getColumnFamily().toString(), - AccumuloByteSerializer.deserialize(key.getRow().getBytes())); - - if (vertexCache != null) { - vertexCache.cache(v); - } - return v; - - } - }; + AccumuloGraphUtils.validateProperty(key, value); + if (globals.getConfig().getAutoIndex() || getIndexedKeys(Vertex.class).contains(key)) { + return globals.getVertexKeyIndexWrapper().getVertices(key, value); } else { - byte[] val = AccumuloByteSerializer.serialize(value); - if (val[0] != AccumuloByteSerializer.SERIALIZABLE) { - BatchScanner scan = getElementBatchScanner(Vertex.class); - scan.fetchColumnFamily(new Text(key)); - - IteratorSetting is = new IteratorSetting(10, "filter", RegExFilter.class); - RegExFilter.setRegexs(is, null, null, null, Pattern.quote(new String(val)), false); - scan.addScanIterator(is); - - return new ScannerIterable(this, scan) { - - @Override - public Vertex next(PeekingIterator> iterator) { - - Entry kv = iterator.next(); - AccumuloVertex v = null; - if (vertexCache != null) { - v = (AccumuloVertex) vertexCache.retrieve(kv.getKey().getRow().toString()); - } - - v = (v == null ? new AccumuloVertex(AccumuloGraph.this, kv.getKey().getRow().toString()) : v); - v.cacheProperty(kv.getKey().getColumnFamily().toString(), - AccumuloByteSerializer.deserialize(kv.getValue().get())); - - if (vertexCache != null) { - vertexCache.cache(v); - } - return v; - } - }; - } else { - // TODO - throw new UnsupportedOperationException("Filtering on binary data not currently supported."); - } + return globals.getVertexWrapper().getVertices(key, value); } } @Override public Edge addEdge(Object id, Vertex outVertex, Vertex inVertex, String label) { - if (label == null) { - throw ExceptionFactory.edgeLabelCanNotBeNull(); - } - String myID; - if (id == null) { - myID = UUID.randomUUID().toString(); - } else { - try { - myID = id.toString(); - } catch (ClassCastException e) { - return null; - } - } - - // TODO we arent suppose to make sure the given edge ID doesn't already - // exist? - - try { - Mutation m = new Mutation(myID); - m.put(LABEL, (inVertex.getId().toString() + IDDELIM + outVertex.getId().toString()).getBytes(), AccumuloByteSerializer.serialize(label)); - edgeBW.addMutation(m); - m = new Mutation(inVertex.getId().toString()); - m.put(INEDGE, (outVertex.getId().toString() + IDDELIM + myID).getBytes(), (IDDELIM + label).getBytes()); - vertexBW.addMutation(m); - m = new Mutation(outVertex.getId().toString()); - m.put(OUTEDGE, (inVertex.getId().toString() + IDDELIM + myID).getBytes(), (IDDELIM + label).getBytes()); - vertexBW.addMutation(m); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - return null; - } - - checkedFlush(); - - AccumuloEdge edge = new AccumuloEdge(this, myID, label, inVertex, outVertex); - if (edgeCache != null) { - edgeCache.cache(edge); - } - return edge; + return ((AccumuloVertex) outVertex).addEdge(id, label, inVertex); } @Override @@ -642,224 +203,63 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { if (id == null) { throw ExceptionFactory.edgeIdCanNotBeNull(); } - String myID = id.toString(); + String idStr = id.toString(); - Edge edge = null; - if (edgeCache != null) { - edge = edgeCache.retrieve(myID); - if (edge != null) { - return edge; - } + Edge edge = globals.getCaches().retrieve(idStr, Edge.class); + if (edge != null) { + return edge; } - Scanner s; - if (!config.getSkipExistenceChecks()) { - s = getElementScanner(Edge.class); - s.setRange(new Range(myID, myID)); - s.fetchColumnFamily(TLABEL); - if (config.getPreloadedProperties() != null) { - for (String x : config.getPreloadedProperties()) { - s.fetchColumnFamily(new Text(x)); - } + edge = new AccumuloEdge(globals, idStr); + + if (!globals.getConfig().getSkipExistenceChecks()) { + // In addition to just an "existence" check, we will also load + // any "preloaded" properties now, which saves us a round-trip + // to Accumulo later. + String[] preload = globals.getConfig().getPreloadedProperties(); + if (preload == null) { + preload = new String[]{}; } - boolean found = s.iterator().hasNext(); - s.close(); - if (!found) { + Map props = globals.getEdgeWrapper() + .readProperties(edge, preload); + // This will be null if the element does not exist, + // in which case return null. + if (props == null) { return null; } - } else { - return new AccumuloEdge(this, myID); - } - // Preload The Properties - edge = new AccumuloEdge(this, myID); - - preloadProperties(s.iterator(), (AccumuloElement) edge); - - if (edgeCache != null) { - edgeCache.cache(edge); - } - return edge; - } - - private void preloadProperties(Iterator> iter, AccumuloElement e) { - while (iter.hasNext()) { - Entry entry = iter.next(); - Key key = entry.getKey(); - String attr = key.getColumnFamily().toString(); - if (SLABEL.equals(attr)) { - if (!key.getColumnQualifier().toString().equals(SEXISTS)) { - AccumuloEdge edge = (AccumuloEdge) e; - String[] ids = key.getColumnQualifier().toString().split("_"); - edge.setInId(ids[0]); - edge.setOutId(ids[1]); - edge.setLabel(entry.getValue().toString()); - } - continue; + for (String key : props.keySet()) { + ((AccumuloElement) edge).setPropertyInMemory(key, props.get(key)); } - Object val = AccumuloByteSerializer.deserialize(entry.getValue().get()); - e.cacheProperty(attr, val); } + + globals.getCaches().cache(edge, Edge.class); + + return edge; } @Override public void removeEdge(Edge edge) { - if (!config.getIndexableGraphDisabled()) - clearIndex(edge.getId()); - - if (edgeCache != null) { - edgeCache.remove(edge.getId()); - } - - Scanner s = getElementScanner(Edge.class); - s.setRange(new Range(edge.getId().toString())); - - Iterator> iter = s.iterator(); - Text inVert = null; - Text outVert = null; - List indexMutations = new ArrayList(); - while (iter.hasNext()) { - Entry e = iter.next(); - Key k = e.getKey(); - if (k.getColumnFamily().equals(TLABEL)) { - String[] ids = k.getColumnQualifier().toString().split(IDDELIM); - inVert = new Text(ids[0]); - outVert = new Text(ids[1]); - } else { - Mutation m = new Mutation(k.getColumnQualifier()); - m.putDelete(k.getColumnFamily(), k.getRow()); - indexMutations.add(m); - } - } - s.close(); - if (inVert == null || outVert == null) { - return; - } - - BatchDeleter edgedeleter = null; - try { - getEdgeIndexWriter().addMutations(indexMutations); - Mutation m = new Mutation(inVert); - m.putDelete(INEDGE, (outVert.toString() + IDDELIM + edge.getId().toString()).getBytes()); - vertexBW.addMutation(m); - m = new Mutation(outVert); - m.putDelete(OUTEDGE, (inVert.toString() + IDDELIM + edge.getId().toString()).getBytes()); - vertexBW.addMutation(m); - m = new Mutation(edge.getId().toString()); - m.putDelete(LABEL, (inVert.toString() + IDDELIM + outVert.toString()).getBytes()); - edgeBW.addMutation(m); - - checkedFlush(); - edgedeleter = config.getConnector().createBatchDeleter(config.getVertexTable(), config.getAuthorizations(), config.getQueryThreads(), - config.getBatchWriterConfig()); - edgedeleter.setRanges(Collections.singleton(new Range(edge.getId().toString()))); - edgedeleter.delete(); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } finally { - if (edgedeleter != null) - edgedeleter.close(); - } + edge.remove(); } @Override public Iterable getEdges() { - BatchScanner scan = getElementBatchScanner(Edge.class); - scan.fetchColumnFamily(TLABEL); - - if (config.getPreloadedProperties() != null) { - for (String x : config.getPreloadedProperties()) { - scan.fetchColumnFamily(new Text(x)); - } - } - - return new ScannerIterable(this, scan) { - - @Override - public Edge next(PeekingIterator> iterator) { - // TODO I dont know if this should work with a batch scanner.... - Entry entry = iterator.next(); - AccumuloEdge edge = new AccumuloEdge(AccumuloGraph.this, entry.getKey().getRow().toString(), AccumuloByteSerializer - .deserialize(entry.getValue().get()).toString()); - - String rowid = entry.getKey().getRow().toString(); - List> vals = new ArrayList>(); - while (iterator.peek() != null && rowid.compareToIgnoreCase(iterator.peek().getKey().getRow().toString()) == 0) { - vals.add(iterator.next()); - } - preloadProperties(vals.iterator(), edge); - if (edgeCache != null) { - edgeCache.cache(edge); - } - return edge; - } - }; + return globals.getEdgeWrapper().getEdges(); } @Override public Iterable getEdges(String key, Object value) { - nullCheckProperty(key, value); + AccumuloGraphUtils.nullCheckProperty(key, value); if (key.equalsIgnoreCase("label")) { - key = SLABEL; + key = Constants.LABEL; } - if (config.getAutoIndex() || getIndexedKeys(Edge.class).contains(key)) { - // Use the index - Scanner s = getEdgeIndexScanner(); - byte[] val = AccumuloByteSerializer.serialize(value); - Text tVal = new Text(val); - s.setRange(new Range(tVal, tVal)); - s.fetchColumnFamily(new Text(key)); - - return new ScannerIterable(this, s) { - @Override - public Edge next(PeekingIterator> iterator) { - AccumuloEdge e = null; - Entry kv = iterator.next(); - if (edgeCache != null) { - e = (AccumuloEdge) edgeCache.retrieve(kv.getKey().getColumnQualifier().toString()); - } - e = (e == null ? new AccumuloEdge(AccumuloGraph.this, kv.getKey().getColumnQualifier().toString()) : e); - - e.cacheProperty(kv.getKey().getColumnFamily().toString(), - AccumuloByteSerializer.deserialize(kv.getKey().getRow().getBytes())); - - if (edgeCache != null) { - edgeCache.cache(e); - } - return e; - } - }; + if (globals.getConfig().getAutoIndex() || getIndexedKeys(Edge.class).contains(key)) { + return globals.getEdgeKeyIndexWrapper().getEdges(key, value); } else { - - BatchScanner scan = getElementBatchScanner(Edge.class); - scan.fetchColumnFamily(new Text(key)); - - byte[] val = AccumuloByteSerializer.serialize(value); - if (val[0] != AccumuloByteSerializer.SERIALIZABLE) { - IteratorSetting is = new IteratorSetting(10, "filter", RegExFilter.class); - RegExFilter.setRegexs(is, null, null, null, Pattern.quote(new String(val)), false); - scan.addScanIterator(is); - - return new ScannerIterable(this, scan) { - - @Override - public Edge next(PeekingIterator> iterator) { - - Key k = iterator.next().getKey(); - - if (k.getColumnFamily().compareTo(AccumuloGraph.TLABEL) == 0) { - String[] vals = k.getColumnQualifier().toString().split(AccumuloGraph.IDDELIM); - return new AccumuloEdge(AccumuloGraph.this, k.getRow().toString(), null, vals[0], vals[1]); - } - return new AccumuloEdge(AccumuloGraph.this, k.getRow().toString()); - } - }; - } else { - // TODO - throw new UnsupportedOperationException("Filtering on binary data not currently supported."); - } + return globals.getEdgeWrapper().getEdges(key, value); } } @@ -872,382 +272,39 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { @Override public void shutdown() { try { - writer.close(); + globals.getMtbw().close(); + globals.getVertexWrapper().close(); + globals.getEdgeWrapper().close(); } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - if (vertexCache != null) { - vertexCache.clear(); - } - - if (edgeCache != null) { - edgeCache.clear(); - } - } - - // public methods not defined by Graph interface, but potentially useful for - // applications that know they are using an AccumuloGraph - public void clear() { - shutdown(); - - try { - TableOperations to; - to = config.getConnector().tableOperations(); - Iterable> it = this.getIndices(); - Iterator> iter = it.iterator(); - while (iter.hasNext()) { - AccumuloIndex in = (AccumuloIndex) iter.next(); - to.delete(in.tableName); - } - - for (String t : config.getTableNames()) { - if (to.exists(t)) { - to.delete(t); - to.create(t); - SortedSet splits = config.getSplits(); - if (splits != null) { - to.addSplits(t, splits); - } - } - } - setupWriters(); - } catch (Exception e) { throw new AccumuloGraphException(e); } - - } - - public void flush() { - try { - writer.flush(); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - } - - public void clearCache() { - vertexCache.clear(); - edgeCache.clear(); - } - - private void checkedFlush() { - if (config.getAutoFlush()) { - flush(); - } - } - - // methods used by AccumuloElement, AccumuloVertex, AccumuloEdge to interact - // with the backing Accumulo data store... - - Pair getProperty(Class type, String id, String key) { - Text colf = null; - if (StringFactory.LABEL.equals(key)) { - colf = AccumuloGraph.TLABEL; - } else { - colf = new Text(key); - } - - Scanner s = getElementScanner(type); - s.setRange(new Range(id)); - s.fetchColumnFamily(colf); - T toRet = null; - Iterator> iter = s.iterator(); - if (iter.hasNext()) { - toRet = AccumuloByteSerializer.deserialize(iter.next().getValue().get()); - } - s.close(); - return new Pair(config.getPropertyCacheTimeout(key), toRet); - } - - void preloadProperties(AccumuloElement element, Class type) { - String[] toPreload = config.getPreloadedProperties(); - if (toPreload == null) { - return; - } - - Scanner s = getElementScanner(type); - s.setRange(new Range(element.getId().toString())); - - // user has requested specific properties... - Text colf = new Text(""); - for (String key : toPreload) { - if (StringFactory.LABEL.equals(key)) { - colf.set(AccumuloGraph.LABEL); - } else { - colf.set(key); - } - s.fetchColumnFamily(colf); - } - - Iterator> iter = s.iterator(); - // Integer timeout = config.getPropertyCacheTimeoutMillis(); // Change this - while (iter.hasNext()) { - Entry entry = iter.next(); - Object val = AccumuloByteSerializer.deserialize(entry.getValue().get()); - element.cacheProperty(entry.getKey().getColumnFamily().toString(), val); - } - s.close(); - } - - Set getPropertyKeys(Class type, String id) { - Scanner s = getElementScanner(type); - s.setRange(new Range(id)); - Set toRet = new HashSet(); - Iterator> iter = s.iterator(); - while (iter.hasNext()) { - Entry e = iter.next(); - Key k = e.getKey(); - String cf = k.getColumnFamily().toString(); - toRet.add(cf); - } - toRet.remove(TINEDGE.toString()); - toRet.remove(TLABEL.toString()); - toRet.remove(TOUTEDGE.toString()); - s.close(); - return toRet; - } - - /** - * Sets the property. Requires a round-trip to Accumulo to see if the property exists - * iff the provided key has an index. Therefore, for best performance, if at - * all possible, create indices after bulk ingest. - * - * @param type - * @param id - * @param key - * @param val - */ - void setProperty(Class type, String id, String key, Object val) { - checkProperty(key, val); - try { - byte[] newByteVal = AccumuloByteSerializer.serialize(val); - Mutation m = null; - - if (config.getAutoIndex() || getIndexedKeys(type).contains(key)) { - BatchWriter bw = getIndexBatchWriter(type); - Object old = getProperty(type, id, key).getSecond(); - if (old != null) { - byte[] oldByteVal = AccumuloByteSerializer.serialize(old); - m = new Mutation(oldByteVal); - m.putDelete(key, id); - bw.addMutation(m); - } - m = new Mutation(newByteVal); - m.put(key.getBytes(), id.getBytes(), EMPTY); - bw.addMutation(m); - checkedFlush(); - } - - m = new Mutation(id); - m.put(key.getBytes(), EMPTY, newByteVal); - getBatchWriter(type).addMutation(m); - - checkedFlush(); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - } - - private BatchWriter getBatchWriter(Class type) { - if (type.equals(Edge.class)) - return edgeBW; - return vertexBW; - } - - private BatchWriter getIndexBatchWriter(Class type) { - if (type.equals(Edge.class)) - return getEdgeIndexWriter(); - return getVertexIndexWriter(); - } - - T removeProperty(Class type, String id, String key) { - if (StringFactory.LABEL.equals(key) || SLABEL.equals(key)) { - throw new AccumuloGraphException("Cannot remove the " + StringFactory.LABEL + " property."); - } - - T obj = (T) getProperty(type, id, key).getSecond(); - try { - if (obj != null) { - byte[] val = AccumuloByteSerializer.serialize(obj); - Mutation m = new Mutation(id); - m.putDelete(key.getBytes(), EMPTY); - BatchWriter bw = getBatchWriter(type); - bw.addMutation(m); - m = new Mutation(val); - m.putDelete(key, id); - getIndexBatchWriter(type).addMutation(m); - checkedFlush(); - } - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - return obj; - } - - Iterable getEdges(String vertexId, Direction direction, String... labels) { - Scanner scan = getElementScanner(Vertex.class); - scan.setRange(new Range(vertexId)); - if (direction.equals(Direction.IN)) { - scan.fetchColumnFamily(TINEDGE); - } else if (direction.equals(Direction.OUT)) { - scan.fetchColumnFamily(TOUTEDGE); - } else { - scan.fetchColumnFamily(TINEDGE); - scan.fetchColumnFamily(TOUTEDGE); - } - if (labels.length > 0) { - applyRegexValueFilter(scan, labels); - } - - return new ScannerIterable(this, scan) { - - @Override - public Edge next(PeekingIterator> iterator) { - // TODO better use of information readily available... - // TODO could also check local cache before creating a new - // instance? - - Entry kv = iterator.next(); - - String[] parts = kv.getKey().getColumnQualifier().toString().split(IDDELIM); - String label = (new String(kv.getValue().get())).split("_")[1]; - if (kv.getKey().getColumnFamily().toString().equalsIgnoreCase(AccumuloGraph.SINEDGE)) { - return new AccumuloEdge(AccumuloGraph.this, parts[1], label, kv.getKey().getRow().toString(), parts[0]); - - } else { - return new AccumuloEdge(AccumuloGraph.this, parts[1], label, parts[0], kv.getKey().getRow().toString()); - - } - } - }; - } - - private void applyRegexValueFilter(Scanner scan, String... labels) { - StringBuilder regex = new StringBuilder(); - for (String lab : labels) { - if (regex.length() != 0) - regex.append("|"); - regex.append(".*_\\Q").append(lab).append("\\E$"); - } - - IteratorSetting is = new IteratorSetting(10, "getEdgeFilter", RegExFilter.class); - RegExFilter.setRegexs(is, null, null, null, regex.toString(), false); - scan.addScanIterator(is); - } - - Iterable getVertices(String vertexId, Direction direction, String... labels) { - Scanner scan = getElementScanner(Vertex.class); - scan.setRange(new Range(vertexId)); - if (direction.equals(Direction.IN)) { - scan.fetchColumnFamily(TINEDGE); - } else if (direction.equals(Direction.OUT)) { - scan.fetchColumnFamily(TOUTEDGE); - } else { - scan.fetchColumnFamily(TINEDGE); - scan.fetchColumnFamily(TOUTEDGE); - } - - if (labels != null && labels.length > 0) { - applyRegexValueFilter(scan, labels); - } - - return new ScannerIterable(this, scan) { - - @Override - public Vertex next(PeekingIterator> iterator) { - // TODO better use of information readily available... - // TODO could also check local cache before creating a new - // instance? - String[] parts = iterator.next().getKey().getColumnQualifier().toString().split(IDDELIM); - AccumuloVertex v = new AccumuloVertex(AccumuloGraph.this, parts[0]); - if (vertexCache != null) - vertexCache.cache(v); - return v; - } - }; - } - - Vertex getEdgeVertex(String edgeId, Direction direction) { - Scanner s = getElementScanner(Edge.class); - try { - s.setRange(new Range(edgeId)); - s.fetchColumnFamily(TLABEL); - Iterator> iter = s.iterator(); - if (!iter.hasNext()) { - return null; - } - String id; - String val = iter.next().getKey().getColumnQualifier().toString(); - if (direction == Direction.IN) { - id = val.split(IDDELIM)[0]; - } else { - id = val.split(IDDELIM)[1]; - } - Vertex v = new AccumuloVertex(this, id); - if (vertexCache != null) - vertexCache.cache(v); - return v; - - } finally { - s.close(); - } - } - - private void nullCheckProperty(String key, Object val) { - if (key == null) { - throw ExceptionFactory.propertyKeyCanNotBeNull(); - } else if (val == null) { - throw ExceptionFactory.propertyValueCanNotBeNull(); - } else if (key.trim().equals(StringFactory.EMPTY_STRING)) { - throw ExceptionFactory.propertyKeyCanNotBeEmpty(); - } - } - - // internal methods used by this class - - private void checkProperty(String key, Object val) { - nullCheckProperty(key, val); - if (key.equals(StringFactory.ID)) { - throw ExceptionFactory.propertyKeyIdIsReserved(); - } else if (key.equals(StringFactory.LABEL)) { - throw ExceptionFactory.propertyKeyLabelIsReservedForEdges(); - } else if (val == null) { - throw ExceptionFactory.propertyValueCanNotBeNull(); - } + globals.getCaches().clear(Vertex.class); + globals.getCaches().clear(Edge.class); } @Override public String toString() { - return "accumulograph"; + return AccumuloGraphConfiguration.ACCUMULO_GRAPH_CLASS.getSimpleName().toLowerCase(); } + @SuppressWarnings("rawtypes") @Override public Index createIndex(String indexName, Class indexClass, Parameter... indexParameters) { if (indexClass == null) { throw ExceptionFactory.classForElementCannotBeNull(); } - if (config.getIndexableGraphDisabled()) + else if (globals.getConfig().getIndexableGraphDisabled()) { throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration"); - - Scanner s = this.getMetadataScanner(); - try { - s.setRange(new Range(indexName, indexName)); - if (s.iterator().hasNext()) - throw ExceptionFactory.indexAlreadyExists(indexName); - - BatchWriter writer = getWriter(config.getMetadataTable()); - Mutation m = new Mutation(indexName); - m.put(indexClass.getSimpleName().getBytes(), EMPTY, EMPTY); - try { - writer.addMutation(m); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - return new AccumuloIndex(indexClass, this, indexName); - } finally { - s.close(); } + + for (Index index : globals.getNamedIndexListWrapper().getIndices()) { + if (index.getIndexName().equals(indexName)) { + throw ExceptionFactory.indexAlreadyExists(indexName); + } + } + + return globals.getNamedIndexListWrapper().createIndex(indexName, indexClass); } @Override @@ -1255,96 +312,62 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { if (indexClass == null) { throw ExceptionFactory.classForElementCannotBeNull(); } - if (config.getIndexableGraphDisabled()) + else if (globals.getConfig().getIndexableGraphDisabled()) { throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration"); - - Scanner scan = getScanner(config.getMetadataTable()); - try { - scan.setRange(new Range(indexName, indexName)); - Iterator> iter = scan.iterator(); - - while (iter.hasNext()) { - Key k = iter.next().getKey(); - if (k.getColumnFamily().toString().equals(indexClass.getSimpleName())) { - return new AccumuloIndex(indexClass, this, indexName); - } else { - throw ExceptionFactory.indexDoesNotSupportClass(indexName, indexClass); - } - } - return null; - } finally { - scan.close(); } + + return globals.getNamedIndexListWrapper().getIndex(indexName, indexClass); } @Override public Iterable> getIndices() { - if (config.getIndexableGraphDisabled()) + if (globals.getConfig().getIndexableGraphDisabled()) { throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration"); - List> toRet = new ArrayList>(); - Scanner scan = getScanner(config.getMetadataTable()); - try { - Iterator> iter = scan.iterator(); - - while (iter.hasNext()) { - Key k = iter.next().getKey(); - toRet.add(new AccumuloIndex(getClass(k.getColumnFamily().toString()), - this, k.getRow().toString())); - } - return toRet; - } catch (Exception e) { - throw new AccumuloGraphException(e); - } finally { - scan.close(); } - } - - private Class getClass(String e) { - if (e.equals("Vertex")) { - return Vertex.class; - } - return Edge.class; + return globals.getNamedIndexListWrapper().getIndices(); } @Override public void dropIndex(String indexName) { - if (config.getIndexableGraphDisabled()) + if (globals.getConfig().getIndexableGraphDisabled()) throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration"); - BatchDeleter deleter = null; - try { - deleter = config.getConnector().createBatchDeleter(config.getMetadataTable(), config.getAuthorizations(), config.getQueryThreads(), - config.getBatchWriterConfig()); - deleter.setRanges(Collections.singleton(new Range(indexName))); - deleter.delete(); - config.getConnector().tableOperations().delete(config.getGraphName() + "_index_" + indexName); - } catch (Exception e) { - throw new AccumuloGraphException(e); - } finally { - if (deleter != null) - deleter.close(); + for (Index index : getIndices()) { + if (index.getIndexName().equals(indexName)) { + globals.getNamedIndexListWrapper().clearIndexNameEntry(indexName, index.getIndexClass()); + + try { + globals.getConfig().getConnector().tableOperations().delete(globals.getConfig() + .getNamedIndexTableName(indexName)); + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + + return; + } } + + throw new AccumuloGraphException("Index does not exist: "+indexName); } @Override public void dropKeyIndex(String key, Class elementClass) { + // TODO Move below to somewhere appropriate. if (elementClass == null) { throw ExceptionFactory.classForElementCannotBeNull(); } + globals.getIndexedKeysListWrapper().clearKeyMetadataEntry(key, elementClass); + String table = null; if (elementClass.equals(Vertex.class)) { - table = config.getVertexIndexTable(); + table = globals.getConfig().getVertexKeyIndexTableName(); } else { - table = config.getEdgeIndexTable(); + table = globals.getConfig().getEdgeKeyIndexTableName(); } - BatchWriter w = getKeyMetadataWriter(); BatchDeleter bd = null; - Mutation m = new Mutation(key); - m.putDelete(elementClass.getSimpleName().getBytes(), EMPTY); try { - bd = config.getConnector().createBatchDeleter(table, config.getAuthorizations(), config.getMaxWriteThreads(), config.getBatchWriterConfig()); - w.addMutation(m); + bd = globals.getConfig().getConnector().createBatchDeleter(table, globals.getConfig().getAuthorizations(), globals.getConfig().getMaxWriteThreads(), globals.getConfig().getBatchWriterConfig()); bd.setRanges(Collections.singleton(new Range())); bd.fetchColumnFamily(new Text(key)); bd.delete(); @@ -1354,81 +377,76 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph { if (bd != null) bd.close(); } - checkedFlush(); + globals.checkedFlush(); } + @SuppressWarnings("rawtypes") @Override public void createKeyIndex(String key, Class elementClass, Parameter... indexParameters) { + // TODO Move below to somewhere appropriate. if (elementClass == null) { throw ExceptionFactory.classForElementCannotBeNull(); } - BatchWriter w = getKeyMetadataWriter(); - Mutation m = new Mutation(key); - m.put(elementClass.getSimpleName().getBytes(), EMPTY, EMPTY); - try { - w.addMutation(m); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - checkedFlush(); - // Re Index Graph - BatchScanner scan = getElementBatchScanner(elementClass); - try { - scan.setRanges(Collections.singleton(new Range())); - scan.fetchColumnFamily(new Text(key)); - Iterator> iter = scan.iterator(); - - BatchWriter bw = getIndexBatchWriter(elementClass); - while (iter.hasNext()) { - Entry entry = iter.next(); - Key k = entry.getKey(); - Value v = entry.getValue(); - Mutation mu = new Mutation(v.get()); - mu.put(k.getColumnFamily().getBytes(), k.getRow().getBytes(), EMPTY); - try { - bw.addMutation(mu); - } catch (MutationsRejectedException e) { - // TODO handle this better. - throw new AccumuloGraphException(e); - } - } - } finally { - scan.close(); - } - checkedFlush(); + // Add key to indexed keys list. + globals.getIndexedKeysListWrapper().writeKeyMetadataEntry(key, elementClass); + globals.checkedFlush(); + // Reindex graph. + globals.getKeyIndexTableWrapper(elementClass).rebuildIndex(key, elementClass); + globals.getVertexKeyIndexWrapper().dump(); + globals.checkedFlush(); } @Override public Set getIndexedKeys(Class elementClass) { - if (elementClass == null) { - throw ExceptionFactory.classForElementCannotBeNull(); - } + return globals.getIndexedKeysListWrapper().getIndexedKeys(elementClass); + } - Scanner s = getKeyMetadataScanner(); + /** + * Clear out this graph. This drops and recreates the backing tables. + */ + public void clear() { + shutdown(); try { - s.fetchColumnFamily(new Text(elementClass.getSimpleName())); - Iterator> iter = s.iterator(); - Set toRet = new HashSet(); - while (iter.hasNext()) { - toRet.add(iter.next().getKey().getRow().toString()); + TableOperations tableOps = globals.getConfig() + .getConnector().tableOperations(); + for (Index index : getIndices()) { + tableOps.delete(((AccumuloIndex) + index).getTableName()); } - return toRet; - } finally { - s.close(); + + for (String table : globals.getConfig().getTableNames()) { + if (tableOps.exists(table)) { + tableOps.delete(table); + tableOps.create(table); + + SortedSet splits = globals.getConfig().getSplits(); + if (splits != null) { + tableOps.addSplits(table, splits); + } + } + } + } catch (Exception e) { + throw new AccumuloGraphException(e); } } public boolean isEmpty() { - for (String t : config.getTableNames()) { - if (getScanner(t).iterator().hasNext()) { - return false; + try { + TableOperations tableOps = globals.getConfig().getConnector().tableOperations(); + for (String table : globals.getConfig().getTableNames()) { + if (tableOps.getMaxRow(table, globals.getConfig().getAuthorizations(), + null, true, null, true) != null) { + return false; + } } - } + return true; - return true; + } catch (Exception e) { + throw new AccumuloGraphException(e); + } } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphConfiguration.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphConfiguration.java index 6f973da..b2b4887 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphConfiguration.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphConfiguration.java @@ -43,6 +43,7 @@ import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.hadoop.io.Text; +import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.IndexableGraph; import com.tinkerpop.blueprints.KeyIndexableGraph; @@ -69,6 +70,11 @@ implements Serializable { private MiniAccumuloCluster accumuloMiniCluster; + /** + * The {@link AccumuloGraph} class. + */ + public static final Class ACCUMULO_GRAPH_CLASS = AccumuloGraph.class; + /** * The fully-qualified class name of the class that implements * the TinkerPop Graph interface. @@ -76,7 +82,7 @@ implements Serializable { * which type to instantiate. */ public static final String ACCUMULO_GRAPH_CLASSNAME = - AccumuloGraph.class.getCanonicalName(); + ACCUMULO_GRAPH_CLASS.getCanonicalName(); /** * An enumeration used by {@link AccumuloGraphConfiguration#setInstanceType(InstanceType)} @@ -922,42 +928,71 @@ implements Serializable { return this; } - public String getVertexTable() { + /** + * Name of vertex table (keyed by vertex id). + * @return + */ + public String getVertexTableName() { return getGraphName() + "_vertex"; } - public String getEdgeTable() { + /** + * Name of edge table (keyed by edge id). + * @return + */ + public String getEdgeTableName() { return getGraphName() + "_edge"; } - public String getKeyMetadataTable() { - return getMetadataTable() + "KEY"; + /** + * Name of vertex key index table (keyed on + * vertex property keys). + * @return + */ + public String getVertexKeyIndexTableName() { + return getGraphName() + "_vertex_key_index"; } - String getVertexIndexTable() { - return getGraphName() + "_vertex_index"; + /** + * Name of edge key index table (keyed on + * edge property keys). + * @return + */ + public String getEdgeKeyIndexTableName() { + return getGraphName() + "_edge_key_index"; } - String getEdgeIndexTable() { - return getGraphName() + "_edge_index"; + /** + * Table of the index with given name (keyed + * on property keys of the given element type). + * @param indexName + * @return + */ + public String getNamedIndexTableName(String indexName) { + return getGraphName() + "_index_" + indexName; } - String getMetadataTable() { - return getGraphName() + "_meta"; + /** + * Table listing the key-indexed properties + * of elements. + * @return + */ + public String getIndexedKeysTableName() { + return getGraphName() + "_indexed_keys"; } - String getKeyVertexIndexTable() { - return getGraphName() + "_vertex_index_key"; - } - - String getKeyEdgeIndexTable() { - return getGraphName() + "_edge_index_key"; + /** + * Table of existing named indexes. + * @return + */ + public String getIndexNamesTableName() { + return getGraphName() + "_index_names"; } List getTableNames() { - return Arrays.asList(getVertexTable(), - getEdgeTable(), getVertexIndexTable(), getEdgeIndexTable(), - getMetadataTable(), getKeyMetadataTable()); + return Arrays.asList(getVertexTableName(), + getEdgeTableName(), getVertexKeyIndexTableName(), getEdgeKeyIndexTableName(), + getIndexNamesTableName(), getIndexedKeysTableName()); } /** diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphUtils.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphUtils.java index 4008da0..8857938 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphUtils.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloGraphUtils.java @@ -15,11 +15,15 @@ package edu.jhuapl.tinkerpop; import java.util.SortedSet; +import java.util.UUID; import org.apache.accumulo.core.client.admin.TableOperations; import org.apache.hadoop.io.Text; -final class AccumuloGraphUtils { +import com.tinkerpop.blueprints.util.ExceptionFactory; +import com.tinkerpop.blueprints.util.StringFactory; + +public final class AccumuloGraphUtils { /** * Create and/or clear existing graph tables for the given configuration. @@ -76,4 +80,45 @@ final class AccumuloGraphUtils { throw new IllegalArgumentException(e); } } + + /** + * Generate an element id. + * @return + */ + public static String generateId() { + return UUID.randomUUID().toString(); + } + + /** + * Ensure that the given key/value don't conflict with + * Blueprints reserved words. + * @param key + * @param value + */ + public static void validateProperty(String key, Object value) { + nullCheckProperty(key, value); + if (key.equals(StringFactory.ID)) { + throw ExceptionFactory.propertyKeyIdIsReserved(); + } else if (key.equals(StringFactory.LABEL)) { + throw ExceptionFactory.propertyKeyLabelIsReservedForEdges(); + } else if (value == null) { + throw ExceptionFactory.propertyValueCanNotBeNull(); + } + } + + /** + * Disallow null keys/values and throw appropriate + * Blueprints exceptions. + * @param key + * @param value + */ + public static void nullCheckProperty(String key, Object value) { + if (key == null) { + throw ExceptionFactory.propertyKeyCanNotBeNull(); + } else if (value == null) { + throw ExceptionFactory.propertyValueCanNotBeNull(); + } else if (key.trim().equals(StringFactory.EMPTY_STRING)) { + throw ExceptionFactory.propertyKeyCanNotBeEmpty(); + } + } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloIndex.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloIndex.java index a0974c6..79f379d 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloIndex.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloIndex.java @@ -15,82 +15,76 @@ package edu.jhuapl.tinkerpop; import java.util.Iterator; -import java.util.Map.Entry; - -import org.apache.accumulo.core.client.BatchWriter; -import org.apache.accumulo.core.client.MutationsRejectedException; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.client.ScannerBase; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Range; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.util.PeekingIterator; -import org.apache.hadoop.io.Text; import com.tinkerpop.blueprints.CloseableIterable; -import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Element; import com.tinkerpop.blueprints.Index; +import com.tinkerpop.blueprints.IndexableGraph; +import edu.jhuapl.tinkerpop.tables.namedindex.NamedIndexTableWrapper; +/** + * Accumulo-based implementation for {@link IndexableGraph}. + * @param + */ public class AccumuloIndex implements Index { - Class indexedType; - AccumuloGraph parent; - String indexName; - String tableName; + private final GlobalInstances globals; + private final Class indexedType; + private final String indexName; + private final NamedIndexTableWrapper indexWrapper; - public AccumuloIndex(Class t, AccumuloGraph parent, String indexName) { - this.indexedType = t; - this.parent = parent; + public AccumuloIndex(GlobalInstances globals, String indexName, Class indexedType) { + this.globals = globals; this.indexName = indexName; - tableName = parent.config.getGraphName() + "_index_" + indexName;// + "_" + - // t; + this.indexedType = indexedType; try { - if (!parent.config.getConnector().tableOperations().exists(tableName)) { - parent.config.getConnector().tableOperations().create(tableName); + if (!globals.getConfig().getConnector() + .tableOperations().exists(getTableName())) { + globals.getConfig().getConnector() + .tableOperations().create(getTableName()); } } catch (Exception e) { - throw new RuntimeException(e); + throw new AccumuloGraphException(e); } + indexWrapper = new NamedIndexTableWrapper(globals, indexedType, indexName); } + @Override public String getIndexName() { return indexName; } + public String getTableName() { + return globals.getConfig().getNamedIndexTableName(indexName); + } + public NamedIndexTableWrapper getWrapper() { + return indexWrapper; + } + @Override + public Class getIndexClass() { + return indexedType; + } + + @Override public void put(String key, Object value, Element element) { - element.setProperty(key, value); - Mutation m = new Mutation(AccumuloByteSerializer.serialize(value)); - m.put(key.getBytes(), element.getId().toString().getBytes(), "".getBytes()); - BatchWriter w = getWriter(); - try { - w.addMutation(m); - w.flush(); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - + indexWrapper.setPropertyForIndex(element, key, value, true); } + @Override public CloseableIterable get(String key, Object value) { - Scanner scan = getScanner(); - byte[] id = AccumuloByteSerializer.serialize(value); - scan.setRange(new Range(new Text(id), new Text(id))); - scan.fetchColumnFamily(new Text(key)); - - return new IndexIterable(parent, scan, indexedType); + return indexWrapper.readElementsFromIndex(key, value); } + @Override public CloseableIterable query(String key, Object query) { throw new UnsupportedOperationException(); - } + @Override public long count(String key, Object value) { CloseableIterable iterable = get(key, value); Iterator iter = iterable.iterator(); @@ -103,75 +97,8 @@ public class AccumuloIndex implements Index { return count; } - public void remove(String key, Object value, Element element) { - Mutation m = new Mutation(AccumuloByteSerializer.serialize(value)); - m.putDelete(key.getBytes(), element.getId().toString().getBytes()); - BatchWriter w = getWriter(); - try { - w.addMutation(m); - w.flush(); - } catch (MutationsRejectedException e) { - e.printStackTrace(); - } - - } - - private BatchWriter getWriter() { - return parent.getWriter(tableName); - } - - private Scanner getScanner() { - return parent.getScanner(tableName); - } - - public class IndexIterable implements CloseableIterable { - AccumuloGraph parent; - ScannerBase scan; - boolean isClosed; - Class indexedType; - - IndexIterable(AccumuloGraph parent, ScannerBase scan, Class t) { - this.scan = scan; - this.parent = parent; - isClosed = false; - indexedType = t; - } - - public Iterator iterator() { - if (!isClosed) { - return new ScannerIterable(parent, scan) { - - @Override - public T next(PeekingIterator> iterator) { - String id = iterator.next() - .getKey().getColumnQualifier().toString(); - // TODO better use of information readily - // available... - if (indexedType.equals(Edge.class)) { - return (T) new AccumuloEdge(parent, id); - } - else { - return (T) new AccumuloVertex(parent, id); - } - } - }.iterator(); - } - else { - return null; - } - } - - public void close() { - if (!isClosed) { - scan.close(); - isClosed = true; - } - } - - } - @Override - public Class getIndexClass() { - return indexedType; + public void remove(String key, Object value, Element element) { + indexWrapper.removePropertyFromIndex(element, key, value); } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/AccumuloVertex.java b/src/main/java/edu/jhuapl/tinkerpop/AccumuloVertex.java index 2b4cbb0..5779eb4 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/AccumuloVertex.java +++ b/src/main/java/edu/jhuapl/tinkerpop/AccumuloVertex.java @@ -14,26 +14,32 @@ */ package edu.jhuapl.tinkerpop; +import java.util.Map; + import com.tinkerpop.blueprints.Direction; import com.tinkerpop.blueprints.Edge; import com.tinkerpop.blueprints.Vertex; import com.tinkerpop.blueprints.VertexQuery; import com.tinkerpop.blueprints.util.DefaultVertexQuery; +import com.tinkerpop.blueprints.util.ExceptionFactory; +/** + * TODO + */ public class AccumuloVertex extends AccumuloElement implements Vertex { - AccumuloVertex(AccumuloGraph parent, String id) { - super(parent, id, Vertex.class); + public AccumuloVertex(GlobalInstances globals, String id) { + super(globals, id, Vertex.class); } @Override public Iterable getEdges(Direction direction, String... labels) { - return parent.getEdges(id, direction, labels); + return globals.getVertexWrapper().getEdges(this, direction, labels); } @Override public Iterable getVertices(Direction direction, String... labels) { - return parent.getVertices(id, direction, labels); + return globals.getVertexWrapper().getVertices(this, direction, labels); } @Override @@ -43,12 +49,72 @@ public class AccumuloVertex extends AccumuloElement implements Vertex { @Override public Edge addEdge(String label, Vertex inVertex) { - return parent.addEdge(null, this, inVertex, label); + return addEdge(null, label, inVertex); + } + + /** + * Add an edge as with {@link #addEdge(String, Vertex)}, + * but with a specified edge id. + * @param id + * @param label + * @param inVertex + * @return + */ + public Edge addEdge(Object id, String label, Vertex inVertex) { + if (label == null) { + throw ExceptionFactory.edgeLabelCanNotBeNull(); + } + if (id == null) { + id = AccumuloGraphUtils.generateId(); + } + + String myID = id.toString(); + + AccumuloEdge edge = new AccumuloEdge(globals, myID, inVertex, this, label); + + // TODO we arent suppose to make sure the given edge ID doesn't already + // exist? + + globals.getEdgeWrapper().writeEdge(edge); + globals.getVertexWrapper().writeEdgeEndpoints(edge); + + globals.checkedFlush(); + + globals.getCaches().cache(edge, Edge.class); + + return edge; } @Override public void remove() { - parent.removeVertex(this); + globals.getCaches().remove(getId(), Vertex.class); + + super.removeElementFromNamedIndexes(); + + // Throw exception if the element does not exist. + if (!globals.getVertexWrapper().elementExists(id)) { + throw ExceptionFactory.vertexWithIdDoesNotExist(getId()); + } + + // Remove properties from key/value indexes. + Map props = globals.getVertexWrapper() + .readAllProperties(this); + + for (String key : props.keySet()) { + globals.getVertexKeyIndexWrapper().removePropertyFromIndex(this, + key, props.get(key)); + } + + // Remove edges incident to this vertex. + for (Edge edge : getEdges(Direction.BOTH)) { + edge.remove(); + } + + globals.checkedFlush(); + + // Get rid of the vertex. + globals.getVertexWrapper().deleteVertex(this); + globals.checkedFlush(); } @Override diff --git a/src/main/java/edu/jhuapl/tinkerpop/Constants.java b/src/main/java/edu/jhuapl/tinkerpop/Constants.java new file mode 100644 index 0000000..386f041 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/Constants.java @@ -0,0 +1,37 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop; + +/** + * Collect up various constants here. + * @author Michael Lieberman + * + */ +public class Constants { + + private Constants() { } + + public static final String ID_DELIM = "__DELIM__"; + + public static final byte[] EMPTY = new byte[0]; + + /** + * Prefixes for various Accumulo entries. + */ + public static final String LABEL = "__LABEL__"; + public static final String IN_EDGE = "__INEDGE__"; + public static final String OUT_EDGE = "__OUTEDGE__"; + public static final String EXISTS = "__EXISTS__"; +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/GlobalInstances.java b/src/main/java/edu/jhuapl/tinkerpop/GlobalInstances.java new file mode 100644 index 0000000..82e5227 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/GlobalInstances.java @@ -0,0 +1,119 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop; + +import org.apache.accumulo.core.client.MultiTableBatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; + +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.cache.ElementCaches; +import edu.jhuapl.tinkerpop.tables.core.EdgeTableWrapper; +import edu.jhuapl.tinkerpop.tables.core.ElementTableWrapper; +import edu.jhuapl.tinkerpop.tables.core.VertexTableWrapper; +import edu.jhuapl.tinkerpop.tables.keyindex.BaseKeyIndexTableWrapper; +import edu.jhuapl.tinkerpop.tables.keyindex.EdgeKeyIndexTableWrapper; +import edu.jhuapl.tinkerpop.tables.keyindex.IndexedKeysListTableWrapper; +import edu.jhuapl.tinkerpop.tables.keyindex.VertexKeyIndexTableWrapper; +import edu.jhuapl.tinkerpop.tables.namedindex.NamedIndexListTableWrapper; + +/** + * Internal class gathering together instances of + * objects needed for various AccumuloGraph components. + */ +public class GlobalInstances { + + private final AccumuloGraphConfiguration config; + private final MultiTableBatchWriter mtbw; + private final ElementCaches caches; + + public GlobalInstances(AccumuloGraphConfiguration config, + MultiTableBatchWriter mtbw, ElementCaches caches) { + this.config = config; + this.mtbw = mtbw; + this.caches = caches; + } + + public AccumuloGraphConfiguration getConfig() { + return config; + } + + public MultiTableBatchWriter getMtbw() { + return mtbw; + } + + public VertexTableWrapper getVertexWrapper() { + return new VertexTableWrapper(this); + } + + public EdgeTableWrapper getEdgeWrapper() { + return new EdgeTableWrapper(this); + } + + public VertexKeyIndexTableWrapper getVertexKeyIndexWrapper() { + return new VertexKeyIndexTableWrapper(this); + } + + public EdgeKeyIndexTableWrapper getEdgeKeyIndexWrapper() { + return new EdgeKeyIndexTableWrapper(this); + } + + public IndexedKeysListTableWrapper getIndexedKeysListWrapper() { + return new IndexedKeysListTableWrapper(this); + } + + public NamedIndexListTableWrapper getNamedIndexListWrapper() { + return new NamedIndexListTableWrapper(this); + } + + public ElementTableWrapper getElementWrapper(Class clazz) { + if (Vertex.class.equals(clazz)) { + return getVertexWrapper(); + } else if (Edge.class.equals(clazz)) { + return getEdgeWrapper(); + } else { + throw new AccumuloGraphException("Unrecognized class: "+clazz); + } + } + + public BaseKeyIndexTableWrapper getKeyIndexTableWrapper(Class clazz) { + if (Vertex.class.equals(clazz)) { + return getVertexKeyIndexWrapper(); + } else if (Edge.class.equals(clazz)) { + return getEdgeKeyIndexWrapper(); + } else { + throw new AccumuloGraphException("Unrecognized class: "+clazz); + } + } + + public ElementCaches getCaches() { + return caches; + } + + /** + * Flush the writer, if autoflush is enabled. + */ + public void checkedFlush() { + if (config.getAutoFlush()) { + try { + mtbw.flush(); + } catch (MutationsRejectedException e) { + throw new AccumuloGraphException(e); + } + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/ScannerIterable.java b/src/main/java/edu/jhuapl/tinkerpop/ScannerIterable.java index 67b2249..f601adb 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/ScannerIterable.java +++ b/src/main/java/edu/jhuapl/tinkerpop/ScannerIterable.java @@ -14,7 +14,6 @@ */ package edu.jhuapl.tinkerpop; -import java.io.Closeable; import java.util.Iterator; import java.util.Map.Entry; @@ -23,15 +22,17 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.util.PeekingIterator; +import com.tinkerpop.blueprints.CloseableIterable; import com.tinkerpop.blueprints.Element; -public abstract class ScannerIterable implements Iterable, Closeable { +/** + * TODO + */ +public abstract class ScannerIterable implements CloseableIterable { - AccumuloGraph parent; - ScannerBase scanner; + private ScannerBase scanner; - ScannerIterable(AccumuloGraph parent, ScannerBase scanner) { - this.parent = parent; + public ScannerIterable(ScannerBase scanner) { this.scanner = scanner; } @@ -55,10 +56,10 @@ public abstract class ScannerIterable implements Iterable, close(); } - class ScannerIterator implements Iterator { - PeekingIterator> iterator; + private class ScannerIterator implements Iterator { + private PeekingIterator> iterator; - ScannerIterator(PeekingIterator> iterator) { + private ScannerIterator(PeekingIterator> iterator) { this.iterator = iterator; } @@ -76,7 +77,5 @@ public abstract class ScannerIterable implements Iterable, public void remove() { throw new UnsupportedOperationException(); } - } - } diff --git a/src/main/java/edu/jhuapl/tinkerpop/ElementCache.java b/src/main/java/edu/jhuapl/tinkerpop/cache/ElementCache.java similarity index 97% rename from src/main/java/edu/jhuapl/tinkerpop/ElementCache.java rename to src/main/java/edu/jhuapl/tinkerpop/cache/ElementCache.java index e9f3c99..00fe7c0 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/ElementCache.java +++ b/src/main/java/edu/jhuapl/tinkerpop/cache/ElementCache.java @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package edu.jhuapl.tinkerpop; +package edu.jhuapl.tinkerpop.cache; import java.util.concurrent.TimeUnit; diff --git a/src/main/java/edu/jhuapl/tinkerpop/cache/ElementCaches.java b/src/main/java/edu/jhuapl/tinkerpop/cache/ElementCaches.java new file mode 100644 index 0000000..b5f0307 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/cache/ElementCaches.java @@ -0,0 +1,77 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.cache; + +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration; +import edu.jhuapl.tinkerpop.AccumuloGraphException; + +/** + * Utility class wrapping element caches. + */ +public class ElementCaches { + private ElementCache vertexCache; + private ElementCache edgeCache; + + public ElementCaches(AccumuloGraphConfiguration config) { + if (config.getVertexCacheEnabled()) { + vertexCache = new ElementCache(config.getVertexCacheSize(), + config.getVertexCacheTimeout()); + } + + if (config.getEdgeCacheEnabled()) { + edgeCache = new ElementCache(config.getEdgeCacheSize(), + config.getEdgeCacheTimeout()); + } + } + + public void cache(T element, Class clazz) { + if (pick(clazz) != null) { + pick(clazz).cache(element); + } + } + + public T retrieve(Object id, Class clazz) { + return pick(clazz) != null ? pick(clazz).retrieve(id) : null; + } + + public void remove(Object id, Class clazz) { + if (pick(clazz) != null) { + pick(clazz).remove(id); + } + } + + public void clear(Class clazz) { + if (pick(clazz) != null) { + pick(clazz).clear(); + } + } + + @SuppressWarnings("unchecked") + private ElementCache pick(Class clazz) { + if (Vertex.class.equals(clazz)) { + return (ElementCache) vertexCache; + } + else if (Edge.class.equals(clazz)) { + return (ElementCache) edgeCache; + } + else { + throw new AccumuloGraphException("Unknown element class: "+clazz); + } + } +} \ No newline at end of file diff --git a/src/main/java/edu/jhuapl/tinkerpop/PropertyCache.java b/src/main/java/edu/jhuapl/tinkerpop/cache/PropertyCache.java similarity index 81% rename from src/main/java/edu/jhuapl/tinkerpop/PropertyCache.java rename to src/main/java/edu/jhuapl/tinkerpop/cache/PropertyCache.java index 47f68fb..3366287 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/PropertyCache.java +++ b/src/main/java/edu/jhuapl/tinkerpop/cache/PropertyCache.java @@ -12,10 +12,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package edu.jhuapl.tinkerpop; +package edu.jhuapl.tinkerpop.cache; import java.util.HashMap; import java.util.Map; +import java.util.Set; + +import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration; /** * Cache for storing element properties. @@ -34,6 +37,14 @@ public class PropertyCache { this.values = new HashMap(); } + public boolean containsKey(String key) { + return values.containsKey(key); + } + + public Set keySet() { + return values.keySet(); + } + public void put(String key, Object value) { Integer timeout = getTimeout(key); @@ -45,6 +56,13 @@ public class PropertyCache { timeout != null ? System.currentTimeMillis() + timeout : null)); } + public void putAll(Map entries) { + for (String key : entries.keySet()) { + put(key, entries.get(key)); + } + } + + @SuppressWarnings("unchecked") public T get(String key) { long now = System.currentTimeMillis(); @@ -71,6 +89,11 @@ public class PropertyCache { values.clear(); } + @Override + public String toString() { + return values.toString(); + } + /** * Return the timeout for the given key. * Checks for a key-specific timeout @@ -106,5 +129,10 @@ public class PropertyCache { public Long getExpiry() { return expiry; } + + @Override + public String toString() { + return "[" + value + ", " + expiry + "]"; + } } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/EdgeInputFormat.java b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/EdgeInputFormat.java index 01b4e73..734fd99 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/EdgeInputFormat.java +++ b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/EdgeInputFormat.java @@ -24,6 +24,7 @@ import edu.jhuapl.tinkerpop.AccumuloByteSerializer; import edu.jhuapl.tinkerpop.AccumuloGraph; import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration; import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration.InstanceType; +import edu.jhuapl.tinkerpop.Constants; public class EdgeInputFormat extends InputFormatBase { @@ -84,10 +85,10 @@ public class EdgeInputFormat extends InputFormatBase { String eid = currentKey.getRow().toString(); String colf = currentKey.getColumnFamily().toString(); switch (colf) { - case AccumuloGraph.SLABEL: + case Constants.LABEL: currentK.set(eid); edge.prepareId(eid); - String[] ids = currentKey.getColumnQualifier().toString().split(parent.IDDELIM); + String[] ids = currentKey.getColumnQualifier().toString().split(Constants.ID_DELIM); edge.setSourceId(ids[1]); edge.setDestId(ids[0]); edge.setLabel(AccumuloByteSerializer.deserialize(entry.getValue().get()).toString()); @@ -109,7 +110,7 @@ public class EdgeInputFormat extends InputFormatBase { public static void setAccumuloGraphConfiguration(Job job, AccumuloGraphConfiguration cfg) throws AccumuloSecurityException { EdgeInputFormat.setConnectorInfo(job, cfg.getUser(), new PasswordToken(cfg.getPassword())); - EdgeInputFormat.setInputTableName(job, cfg.getEdgeTable()); + EdgeInputFormat.setInputTableName(job, cfg.getEdgeTableName()); if (cfg.getInstanceType().equals(InstanceType.Mock)) { EdgeInputFormat.setMockInstance(job, cfg.getInstanceName()); } else { diff --git a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/ElementOutputFormat.java b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/ElementOutputFormat.java index a0fb5eb..2db59b5 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/ElementOutputFormat.java +++ b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/ElementOutputFormat.java @@ -92,9 +92,9 @@ public class ElementOutputFormat extends OutputFormat { try { if (bw == null) { if (ele instanceof MapReduceVertex) { - bw = config.getConnector().createBatchWriter(config.getVertexTable(), config.getBatchWriterConfig()); + bw = config.getConnector().createBatchWriter(config.getVertexTableName(), config.getBatchWriterConfig()); } else { - bw = config.getConnector().createBatchWriter(config.getEdgeTable(), config.getBatchWriterConfig()); + bw = config.getConnector().createBatchWriter(config.getEdgeTableName(), config.getBatchWriterConfig()); } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/MapReduceElement.java b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/MapReduceElement.java index 2b5a9e6..0a5fb81 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/MapReduceElement.java +++ b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/MapReduceElement.java @@ -64,6 +64,7 @@ public abstract class MapReduceElement implements Element, WritableComparable T getProperty(String key) { diff --git a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/VertexInputFormat.java b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/VertexInputFormat.java index 2cee941..7813788 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/mapreduce/VertexInputFormat.java +++ b/src/main/java/edu/jhuapl/tinkerpop/mapreduce/VertexInputFormat.java @@ -24,6 +24,7 @@ import edu.jhuapl.tinkerpop.AccumuloByteSerializer; import edu.jhuapl.tinkerpop.AccumuloGraph; import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration; import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration.InstanceType; +import edu.jhuapl.tinkerpop.Constants; public class VertexInputFormat extends InputFormatBase { static AccumuloGraphConfiguration conf; @@ -85,17 +86,17 @@ public class VertexInputFormat extends InputFormatBase { String vid = currentKey.getRow().toString(); String colf = currentKey.getColumnFamily().toString(); switch (colf) { - case AccumuloGraph.SLABEL: + case Constants.LABEL: currentK.set(vid); vertex.prepareId(vid); break; - case AccumuloGraph.SINEDGE: - String[] parts = currentKey.getColumnQualifier().toString().split(AccumuloGraph.IDDELIM); + case Constants.IN_EDGE: + String[] parts = currentKey.getColumnQualifier().toString().split(Constants.ID_DELIM); String label = new String(entry.getValue().get()); vertex.prepareEdge(parts[1], parts[0], label, vid); break; - case AccumuloGraph.SOUTEDGE: - parts = currentKey.getColumnQualifier().toString().split(AccumuloGraph.IDDELIM); + case Constants.OUT_EDGE: + parts = currentKey.getColumnQualifier().toString().split(Constants.ID_DELIM); label = new String(entry.getValue().get()); vertex.prepareEdge(parts[1], vid, label, parts[0]); break; @@ -116,7 +117,7 @@ public class VertexInputFormat extends InputFormatBase { public static void setAccumuloGraphConfiguration(Job job, AccumuloGraphConfiguration cfg) throws AccumuloSecurityException { VertexInputFormat.setConnectorInfo(job, cfg.getUser(), new PasswordToken(cfg.getPassword())); - VertexInputFormat.setInputTableName(job, cfg.getVertexTable()); + VertexInputFormat.setInputTableName(job, cfg.getVertexTableName()); if (cfg.getInstanceType().equals(InstanceType.Mock)) { VertexInputFormat.setMockInstance(job, cfg.getInstanceName()); } else { diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutator.java new file mode 100644 index 0000000..098a619 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutator.java @@ -0,0 +1,22 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator; + +import org.apache.accumulo.core.data.Mutation; + +public interface Mutator { + + public Iterable create(); +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutators.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutators.java new file mode 100644 index 0000000..4589bc4 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/Mutators.java @@ -0,0 +1,53 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator; + +import java.util.LinkedList; +import java.util.List; + +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.data.Range; + +import com.tinkerpop.blueprints.Element; + +import edu.jhuapl.tinkerpop.AccumuloGraphException; + +public class Mutators { + + public static void apply(BatchWriter writer, Mutator mut) { + try { + writer.addMutations(mut.create()); + } catch (MutationsRejectedException e) { + throw new AccumuloGraphException(e); + } + } + + public static void deleteElementRanges(BatchDeleter deleter, Element... elements) { + List ranges = new LinkedList(); + + for (Element element : elements) { + ranges.add(new Range(element.getId().toString())); + } + deleter.setRanges(ranges); + + try { + deleter.delete(); + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/BaseEdgeMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/BaseEdgeMutator.java new file mode 100644 index 0000000..c9c84c1 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/BaseEdgeMutator.java @@ -0,0 +1,41 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.edge; + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; + +import edu.jhuapl.tinkerpop.mutator.Mutator; + +public abstract class BaseEdgeMutator implements Mutator { + + protected final String id; + protected final String outVertexId; + protected final String inVertexId; + protected final String label; + + public BaseEdgeMutator(Edge edge) { + this(edge.getId().toString(), + edge.getVertex(Direction.OUT).getId().toString(), + edge.getVertex(Direction.IN).getId().toString(), + edge.getLabel()); + } + public BaseEdgeMutator(String id, String outVertexId, String inVertexId, String label) { + this.id = id; + this.outVertexId = outVertexId; + this.inVertexId = inVertexId; + this.label = label; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeEndpointsMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeEndpointsMutator.java new file mode 100644 index 0000000..0e8d425 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeEndpointsMutator.java @@ -0,0 +1,78 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.edge; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import com.tinkerpop.blueprints.Edge; + +import edu.jhuapl.tinkerpop.Constants; + +public class EdgeEndpointsMutator { + + private EdgeEndpointsMutator() { + } + + public static class Add extends BaseEdgeMutator { + + public Add(Edge edge) { + super(edge); + } + + public Add(String id, String outVertexId, String inVertexId, String label) { + super(id, outVertexId, inVertexId, label); + } + + @Override + public Iterable create() { + Mutation in = new Mutation(inVertexId); + in.put(Constants.IN_EDGE.getBytes(), + (outVertexId + Constants.ID_DELIM + id).getBytes(), + (Constants.ID_DELIM + label).getBytes()); + + Mutation out = new Mutation(outVertexId); + out.put(Constants.OUT_EDGE.getBytes(), + (inVertexId + Constants.ID_DELIM + id).getBytes(), + (Constants.ID_DELIM + label).getBytes()); + + return Lists.newArrayList(in, out); + } + } + + public static class Delete extends BaseEdgeMutator { + + public Delete(Edge edge) { + super(edge); + } + + public Delete(String id, String outVertexId, String inVertexId, String label) { + super(id, outVertexId, inVertexId, label); + } + + @Override + public Iterable create() { + Mutation in = new Mutation(inVertexId); + in.putDelete(Constants.IN_EDGE.getBytes(), + (outVertexId + Constants.ID_DELIM + id).getBytes()); + + Mutation out = new Mutation(outVertexId); + out.putDelete(Constants.OUT_EDGE.getBytes(), + (inVertexId + Constants.ID_DELIM + id).getBytes()); + + return Lists.newArrayList(in, out); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeMutator.java new file mode 100644 index 0000000..25da092 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/edge/EdgeMutator.java @@ -0,0 +1,67 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.edge; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import com.tinkerpop.blueprints.Edge; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.Constants; + +public final class EdgeMutator { + + public static class Add extends BaseEdgeMutator { + + public Add(Edge edge) { + super(edge); + } + + public Add(String id, String outVertexId, String inVertexId, String label) { + super(id, outVertexId, inVertexId, label); + } + + @Override + public Iterable create() { + Mutation m = new Mutation(id); + m.put(Constants.LABEL.getBytes(), + (inVertexId + Constants.ID_DELIM + outVertexId).getBytes(), + AccumuloByteSerializer.serialize(label)); + + return Lists.newArrayList(m); + } + } + + public static class Delete extends BaseEdgeMutator { + + public Delete(Edge edge) { + super(edge); + } + + public Delete(String id, String outVertexId, String inVertexId, String label) { + super(id, outVertexId, inVertexId, label); + } + + @Override + public Iterable create() { + Mutation m = new Mutation(id); + m.putDelete(Constants.LABEL.getBytes(), + (inVertexId + Constants.ID_DELIM + outVertexId).getBytes()); + + return Lists.newArrayList(m); + } + } +} \ No newline at end of file diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexMetadataMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexMetadataMutator.java new file mode 100644 index 0000000..dc53b3b --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexMetadataMutator.java @@ -0,0 +1,68 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.index; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import com.tinkerpop.blueprints.Element; + +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.mutator.Mutator; + +/** + * Mutators for index metadata table entries. + */ +public class IndexMetadataMutator { + + private IndexMetadataMutator() { } + + public static class Add implements Mutator { + + private final String key; + private final Class clazz; + + public Add(String key, Class clazz) { + this.key = key; + this.clazz = clazz; + } + + @Override + public Iterable create() { + Mutation m = new Mutation(key); + m.put(clazz.getName().getBytes(), + Constants.EMPTY, Constants.EMPTY); + return Lists.newArrayList(m); + } + } + + public static class Delete implements Mutator { + + private final String key; + private final Class clazz; + + public Delete(String indexName, Class clazz) { + this.key = indexName; + this.clazz = clazz; + } + + @Override + public Iterable create() { + Mutation m = new Mutation(key); + m.putDelete(clazz.getName().getBytes(), Constants.EMPTY); + return Lists.newArrayList(m); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexValueMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexValueMutator.java new file mode 100644 index 0000000..f921941 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/index/IndexValueMutator.java @@ -0,0 +1,75 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.index; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import com.tinkerpop.blueprints.Element; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.mutator.Mutator; + +/** + * Mutators for vertex/edge index tables. + */ +public class IndexValueMutator { + + private IndexValueMutator() { } + + public static class Add implements Mutator { + + private final Element element; + private final String key; + private final Object value; + + public Add(Element element, String key, Object value) { + this.element = element; + this.key = key; + this.value = value; + } + + @Override + public Iterable create() { + byte[] bytes = AccumuloByteSerializer.serialize(value); + Mutation m = new Mutation(bytes); + m.put(key.getBytes(), element.getId().toString() + .getBytes(), Constants.EMPTY); + return Lists.newArrayList(m); + } + } + + public static class Delete implements Mutator { + + private final Element element; + private final String key; + private final Object value; + + public Delete(Element element, String key, Object value) { + this.element = element; + this.key = key; + this.value = value; + } + + @Override + public Iterable create() { + byte[] bytes = AccumuloByteSerializer.serialize(value); + Mutation m = new Mutation(bytes); + m.putDelete(key, element.getId().toString()); + return Lists.newArrayList(m); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/property/BasePropertyMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/BasePropertyMutator.java new file mode 100644 index 0000000..8bc5601 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/BasePropertyMutator.java @@ -0,0 +1,28 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.property; + +import edu.jhuapl.tinkerpop.mutator.Mutator; + +public abstract class BasePropertyMutator implements Mutator { + + protected final String id; + protected final String key; + + public BasePropertyMutator(String id, String key) { + this.id = id; + this.key = key; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/property/ClearPropertyMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/ClearPropertyMutator.java new file mode 100644 index 0000000..efefdfb --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/ClearPropertyMutator.java @@ -0,0 +1,34 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.property; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import edu.jhuapl.tinkerpop.Constants; + +public class ClearPropertyMutator extends BasePropertyMutator { + + public ClearPropertyMutator(String id, String key) { + super(id, key); + } + + @Override + public Iterable create() { + Mutation m = new Mutation(id); + m.putDelete(key.getBytes(), Constants.EMPTY); + return Lists.newArrayList(m); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/property/WritePropertyMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/WritePropertyMutator.java new file mode 100644 index 0000000..1567831 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/property/WritePropertyMutator.java @@ -0,0 +1,39 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.property; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.Constants; + +public class WritePropertyMutator extends BasePropertyMutator { + + private final Object value; + + public WritePropertyMutator(String id, String key, Object value) { + super(id, key); + this.value = value; + } + + @Override + public Iterable create() { + byte[] bytes = AccumuloByteSerializer.serialize(value); + Mutation m = new Mutation(id); + m.put(key.getBytes(), Constants.EMPTY, bytes); + return Lists.newArrayList(m); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/mutator/vertex/AddVertexMutator.java b/src/main/java/edu/jhuapl/tinkerpop/mutator/vertex/AddVertexMutator.java new file mode 100644 index 0000000..c506af9 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/mutator/vertex/AddVertexMutator.java @@ -0,0 +1,38 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.mutator.vertex; + +import org.apache.accumulo.core.data.Mutation; + +import com.google.common.collect.Lists; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.mutator.Mutator; + +public final class AddVertexMutator implements Mutator { + + private final String id; + + public AddVertexMutator(String id) { + this.id = id; + } + + @Override + public Iterable create() { + Mutation m = new Mutation(id); + m.put(Constants.LABEL.getBytes(), + Constants.EXISTS.getBytes(), Constants.EMPTY); + return Lists.newArrayList(m); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeIndexParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeIndexParser.java new file mode 100644 index 0000000..d8ee0b0 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeIndexParser.java @@ -0,0 +1,33 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import edu.jhuapl.tinkerpop.AccumuloEdge; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * Edge-specific index parser. + */ +public class EdgeIndexParser extends ElementIndexParser { + + public EdgeIndexParser(GlobalInstances globals) { + super(globals); + } + + @Override + protected AccumuloEdge instantiate(String id) { + return new AccumuloEdge(globals, id); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeParser.java new file mode 100644 index 0000000..45b6f5c --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/EdgeParser.java @@ -0,0 +1,70 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloEdge; +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * TODO + */ +public class EdgeParser extends ElementParser { + + public EdgeParser(GlobalInstances globals) { + super(globals); + } + + @Override + public AccumuloEdge parse(String id, Iterable> entries) { + AccumuloEdge edge = makeEdge(id, entries); + setInMemoryProperties(edge, entries); + return edge; + } + + /** + * Make and return an edge object. If the entries + * contain label/endpoint information, set those too. + * @param id + * @param entries + * @return + */ + private AccumuloEdge makeEdge(String id, Iterable> entries) { + for (Entry entry : entries) { + String cf = entry.getKey().getColumnFamily().toString(); + if (Constants.LABEL.equals(cf)) { + String cq = entry.getKey().getColumnQualifier().toString(); + String[] parts = cq.split(Constants.ID_DELIM); + String inVertexId = parts[0]; + String outVertexId = parts[1]; + String label = AccumuloByteSerializer.deserialize(entry.getValue().get()); + return new AccumuloEdge(globals, id, + new AccumuloVertex(globals, inVertexId), + new AccumuloVertex(globals, outVertexId), label); + } + } + + // This should not happen. + throw new AccumuloGraphException("Unable to parse edge from entries"); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/ElementIndexParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/ElementIndexParser.java new file mode 100644 index 0000000..c316803 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/ElementIndexParser.java @@ -0,0 +1,72 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloElement; +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * Parser for elements based on an index table. + */ +public abstract class ElementIndexParser + implements EntryParser { + + protected final GlobalInstances globals; + + public ElementIndexParser(GlobalInstances globals) { + this.globals = globals; + } + + @Override + public T parse(Iterable> entries) { + Iterator> it = entries.iterator(); + + if (it.hasNext()) { + Entry entry = it.next(); + + if (it.hasNext()) { + throw new AccumuloGraphException("Unexpected multiple entries for index table"); + } + + String id = entry.getKey().getColumnQualifierData().toString(); + T element = instantiate(id); + + // While we're here, read the property key/value. + String key = entry.getKey().getColumnFamily().toString(); + Object value = AccumuloByteSerializer.deserialize(entry.getKey().getRow().getBytes()); + element.setPropertyInMemory(key, value); + + return element; + } + else { + throw new AccumuloGraphException("No index table entries found"); + } + } + + /** + * Instantiate an object to be returned. + * @param id + * @return + */ + protected abstract T instantiate(String id); +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/ElementParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/ElementParser.java new file mode 100644 index 0000000..5a67351 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/ElementParser.java @@ -0,0 +1,64 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import edu.jhuapl.tinkerpop.AccumuloElement; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * TODO + */ +public abstract class ElementParser implements EntryParser { + + protected GlobalInstances globals; + + public ElementParser(GlobalInstances globals) { + this.globals = globals; + } + + @Override + public T parse(Iterable> entries) { + String id = entries.iterator().next().getKey().getRow().toString(); + return parse(id, entries); + } + + /** + * Given the element id and set of entries, + * create the type of element. This can leverage + * the property loader, etc. + * @param id + * @param entries + * @return + */ + public abstract T parse(String id, Iterable> entries); + + /** + * Parse out the property entries and set them for the given element. + * @param element + * @param entries + */ + protected void setInMemoryProperties(T element, Iterable> entries) { + Map props = new PropertyParser().parse(entries); + for (String key : props.keySet()) { + element.setPropertyInMemory(key, props.get(key)); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/EntryParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/EntryParser.java new file mode 100644 index 0000000..6330491 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/EntryParser.java @@ -0,0 +1,28 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +/** + * TODO + */ +public interface EntryParser { + + public T parse(Iterable> entries); +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItem.java b/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItem.java new file mode 100644 index 0000000..580a684 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItem.java @@ -0,0 +1,44 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.IndexableGraph; +import com.tinkerpop.blueprints.KeyIndexableGraph; + +/** + * An indexed item. For {@link IndexableGraph}, + * the key is the index name. For {@link KeyIndexableGraph} + * the key is the indexed key. + * @author Michael Lieberman + * + */ +public class IndexedItem { + private final String key; + private final Class elementClass; + + public IndexedItem(String key, Class elementClass) { + this.key = key; + this.elementClass = elementClass; + } + + public String getKey() { + return key; + } + + public Class getElementClass() { + return elementClass; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItemsListParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItemsListParser.java new file mode 100644 index 0000000..8407be7 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/IndexedItemsListParser.java @@ -0,0 +1,91 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.IndexableGraph; +import com.tinkerpop.blueprints.KeyIndexableGraph; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloGraphException; + +/** + * Entry parser for index metadata. The format + * is the same for both {@link IndexableGraph} + * and {@link KeyIndexableGraph} functionality. + * For the former, this parser returns names of + * indexes. For the latter, the parser returns + * indexed keys. + */ +public class IndexedItemsListParser implements EntryParser> { + + private final Class elementClass; + + /** + * Constructor to return all items regardless + * of element class. + */ + public IndexedItemsListParser() { + this(Element.class); + } + + /** + * Create a parser for items of the specified element class. + * This may be Vertex, Edge, or Element. + * @param elementClass + */ + public IndexedItemsListParser(Class elementClass) { + // Validate element class. + if (!Vertex.class.equals(elementClass) && + !Edge.class.equals(elementClass) && + !Element.class.equals(elementClass)) { + throw new IllegalArgumentException("elementClass must be Vertex, Edge or Element"); + } + this.elementClass = elementClass; + } + + @SuppressWarnings("unchecked") + @Override + public List parse(Iterable> entries) { + List items = new ArrayList(); + + for (Entry entry : entries) { + Class clazz; + try { + clazz = (Class) Class.forName(entry.getKey() + .getColumnFamily().toString()); + } catch (ClassNotFoundException e) { + throw new AccumuloGraphException(e); + } + + if (Element.class.equals(elementClass) || + elementClass.equals(clazz)) { + IndexedItem item = new IndexedItem(entry.getKey() + .getRow().toString(), clazz); + items.add(item); + } + } + + return items; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/PropertyParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/PropertyParser.java new file mode 100644 index 0000000..2ca4996 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/PropertyParser.java @@ -0,0 +1,66 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Value; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.Constants; + +/** + * TODO + */ +public class PropertyParser implements EntryParser> { + + @Override + public Map parse(Iterable> entries) { + Map props = null; + + for (Entry entry : entries) { + if (props == null) { + props = new HashMap(); + } + + Key key = entry.getKey(); + + if (!isMetaKey(key)) { + String attr = key.getColumnFamily().toString(); + Object value = AccumuloByteSerializer.deserialize(entry.getValue().get()); + props.put(attr, value); + } + } + + return props; + } + + /** + * Test whether the given Accumulo key represents a + * metadata key (e.g. existence, edge endpoint, etc), + * rather than a property. + * @param key + * @return + */ + private static boolean isMetaKey(Key key) { + String cf = key.getColumnFamily().toString(); + return Constants.LABEL.equals(cf) || + Constants.IN_EDGE.equals(cf) || + Constants.OUT_EDGE.equals(cf); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/parser/VertexIndexParser.java b/src/main/java/edu/jhuapl/tinkerpop/parser/VertexIndexParser.java new file mode 100644 index 0000000..dda2e2f --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/VertexIndexParser.java @@ -0,0 +1,33 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.parser; + +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * Vertex-specific index parser. + */ +public class VertexIndexParser extends ElementIndexParser { + + public VertexIndexParser(GlobalInstances globals) { + super(globals); + } + + @Override + protected AccumuloVertex instantiate(String id) { + return new AccumuloVertex(globals, id); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/EntryLocation.java b/src/main/java/edu/jhuapl/tinkerpop/parser/VertexParser.java similarity index 58% rename from src/main/java/edu/jhuapl/tinkerpop/EntryLocation.java rename to src/main/java/edu/jhuapl/tinkerpop/parser/VertexParser.java index 9dc1913..2f546a4 100644 --- a/src/main/java/edu/jhuapl/tinkerpop/EntryLocation.java +++ b/src/main/java/edu/jhuapl/tinkerpop/parser/VertexParser.java @@ -12,29 +12,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package edu.jhuapl.tinkerpop; +package edu.jhuapl.tinkerpop.parser; import java.util.Map.Entry; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -public enum EntryLocation { +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.GlobalInstances; - Row, ColF, ColQ, Value; +/** + * TODO + */ +public class VertexParser extends ElementParser { - public String extract(Entry entry) { - switch (this) { - case Row: - return entry.getKey().getRow().toString(); - case ColF: - return entry.getKey().getColumnFamily().toString(); - case ColQ: - return entry.getKey().getColumnQualifier().toString(); - case Value: - return new String(entry.getValue().get()); - default: - throw new AccumuloGraphException("Unexpected type: " + this); - } + public VertexParser(GlobalInstances globals) { + super(globals); + } + + @Override + public AccumuloVertex parse(String id, Iterable> entries) { + AccumuloVertex vertex = new AccumuloVertex(globals, id); + setInMemoryProperties(vertex, entries); + return vertex; } } diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexValuesTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexValuesTableWrapper.java new file mode 100644 index 0000000..c69037c --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexValuesTableWrapper.java @@ -0,0 +1,178 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.core.util.PeekingIterator; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.CloseableIterable; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.IndexableGraph; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloElement; +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.AccumuloGraphUtils; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.ScannerIterable; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.mutator.index.IndexValueMutator; +import edu.jhuapl.tinkerpop.parser.EdgeIndexParser; +import edu.jhuapl.tinkerpop.parser.ElementIndexParser; +import edu.jhuapl.tinkerpop.parser.VertexIndexParser; + +/** + * Wrapper around index tables containing properties + * and values. + */ +public abstract class BaseIndexValuesTableWrapper extends BaseTableWrapper { + + protected final Class elementType; + + protected BaseIndexValuesTableWrapper(GlobalInstances globals, + Class elementType, String tableName) { + super(globals, tableName); + this.elementType = elementType; + } + + /** + * Return class of this index. + * @return + */ + public Class getElementType() { + return elementType; + } + + /** + * Add the property to this index, if autoindexing is enabled + * and/or the given key has indexing enabled. + * @param element + * @param key + * @param value + */ + public void setPropertyForIndex(Element element, String key, Object value) { + setPropertyForIndex(element, key, value, false); + } + + /** + * Add the property to this index. + * + *

Note that this requires a round-trip to Accumulo to see + * if the property exists if the provided key has an index. + * So for best performance, create indices after bulk ingest. + *

If the force parameter is true, set the property regardless + * of whether indexing is enabled for the given key. This is needed + * for {@link IndexableGraph} operations. + * @param element + * @param key + * @param value + * @param force + */ + public void setPropertyForIndex(Element element, String key, Object value, + boolean force) { + AccumuloGraphUtils.validateProperty(key, value); + if (force || globals.getConfig().getAutoIndex() || + globals.getIndexedKeysListWrapper() + .getIndexedKeys(elementType).contains(key)) { + BatchWriter writer = getWriter(); + + Object oldValue = element.getProperty(key); + if (oldValue != null && !oldValue.equals(value)) { + Mutators.apply(writer, new IndexValueMutator.Delete(element, key, oldValue)); + } + + Mutators.apply(writer, new IndexValueMutator.Add(element, key, value)); + globals.checkedFlush(); + } + } + + /** + * Remove property from the index. + * @param element + * @param key + * @param value + */ + public void removePropertyFromIndex(Element element, String key, Object value) { + if (value != null) { + Mutators.apply(getWriter(), new IndexValueMutator.Delete(element, key, value)); + globals.checkedFlush(); + } + } + + /** + * Get elements with the key/value pair. + * @param key + * @param value + * @return + */ + @SuppressWarnings("unchecked") + public CloseableIterable readElementsFromIndex(String key, Object value) { + Scanner scan = getScanner(); + byte[] id = AccumuloByteSerializer.serialize(value); + scan.setRange(Range.exact(new Text(id))); + scan.fetchColumnFamily(new Text(key)); + + final ElementIndexParser parser = + Vertex.class.equals(elementType) ? new VertexIndexParser(globals) : + new EdgeIndexParser(globals); + + return new ScannerIterable(scan) { + @Override + public T next(PeekingIterator> iterator) { + return (T) parser.parse(Arrays.asList(iterator.next())); + } + }; + } + + /** + * Remove the given element's properties from the index. + * @param element + */ + public void removeElementFromIndex(Element element) { + BatchDeleter deleter = null; + + try { + deleter = getDeleter(); + deleter.setRanges(Collections.singleton(new Range())); + + IteratorSetting is = new IteratorSetting(10, "getEdgeFilter", RegExFilter.class); + RegExFilter.setRegexs(is, null, null, + "^"+Pattern.quote(element.getId().toString())+"$", null, false); + deleter.addScanIterator(is); + deleter.delete(); + deleter.close(); + } catch (Exception e) { + throw new AccumuloGraphException(e); + } finally { + if (deleter != null) { + deleter.close(); + } + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexedItemsListTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexedItemsListTableWrapper.java new file mode 100644 index 0000000..7ca67c6 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseIndexedItemsListTableWrapper.java @@ -0,0 +1,42 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables; + +import com.tinkerpop.blueprints.Element; + +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.mutator.index.IndexMetadataMutator; + +/** + * Wraps the metadata tables which stores information + * about which property keys are indexed for different + * graph types. + */ +public abstract class BaseIndexedItemsListTableWrapper extends BaseTableWrapper { + + protected BaseIndexedItemsListTableWrapper(GlobalInstances globals, + String tableName) { + super(globals, tableName); + } + + protected void writeEntry(String key, Class clazz) { + Mutators.apply(getWriter(), new IndexMetadataMutator.Add(key, clazz)); + } + + protected void clearEntry(String key, Class clazz) { + Mutators.apply(getWriter(), new IndexMetadataMutator.Delete(key, clazz)); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/BaseTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseTableWrapper.java new file mode 100644 index 0000000..018b7e2 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/BaseTableWrapper.java @@ -0,0 +1,90 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables; + +import java.util.Collections; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; + +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.GlobalInstances; + +/** + * Table wrapper with common functionality. + */ +public abstract class BaseTableWrapper { + + protected GlobalInstances globals; + private String tableName; + + public BaseTableWrapper(GlobalInstances globals, String tableName) { + this.globals = globals; + this.tableName = tableName; + } + + protected Scanner getScanner() { + try { + return globals.getConfig().getConnector().createScanner(tableName, + globals.getConfig().getAuthorizations()); + + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + } + + protected BatchScanner getBatchScanner() { + try { + BatchScanner scanner = globals.getConfig().getConnector().createBatchScanner(tableName, + globals.getConfig().getAuthorizations(), globals.getConfig().getQueryThreads()); + scanner.setRanges(Collections.singletonList(new Range())); + return scanner; + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + } + + protected BatchWriter getWriter() { + try { + return globals.getMtbw().getBatchWriter(tableName); + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + } + + protected BatchDeleter getDeleter() { + try { + return globals.getConfig().getConnector().createBatchDeleter(tableName, + globals.getConfig().getAuthorizations(), globals.getConfig().getMaxWriteThreads(), + globals.getConfig().getBatchWriterConfig()); + } catch (Exception e) { + throw new AccumuloGraphException(e); + } + } + + public void dump() { + System.out.println("Dump of table "+tableName+":"); + Scanner s = getScanner(); + for (Entry entry : s) { + System.out.println(" "+entry); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/core/EdgeTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/core/EdgeTableWrapper.java new file mode 100644 index 0000000..c143492 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/core/EdgeTableWrapper.java @@ -0,0 +1,176 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.core; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.core.util.PeekingIterator; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.Edge; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloEdge; +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.AccumuloGraphUtils; +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.ScannerIterable; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.mutator.edge.EdgeMutator; +import edu.jhuapl.tinkerpop.parser.EdgeParser; + + +/** + * Wrapper around {@link Edge} tables. + */ +public class EdgeTableWrapper extends ElementTableWrapper { + + public EdgeTableWrapper(GlobalInstances globals) { + super(globals, globals.getConfig().getEdgeTableName()); + } + + /** + * Write the given edge to the edge table. Does not + * currently write the edge's properties. + * + *

Note: This only adds the edge information. Vertex + * endpoint information needs to be written to the vertex + * table via {@link VertexTableWrapper}. + * @param edge + */ + public void writeEdge(Edge edge) { + Mutators.apply(getWriter(), new EdgeMutator.Add(edge)); + globals.checkedFlush(); + } + + public void deleteEdge(Edge edge) { + Mutators.apply(getWriter(), new EdgeMutator.Delete(edge)); + globals.checkedFlush(); + } + + public Iterable getEdges() { + Scanner scan = getScanner(); + scan.fetchColumnFamily(new Text(Constants.LABEL)); + + if (globals.getConfig().getPreloadedProperties() != null) { + for (String key : globals.getConfig().getPreloadedProperties()) { + scan.fetchColumnFamily(new Text(key)); + } + } + + final EdgeParser parser = new EdgeParser(globals); + + return new ScannerIterable(scan) { + @Override + public Edge next(PeekingIterator> iterator) { + // TODO could also check local cache before creating a new instance? + + String rowId = iterator.peek().getKey().getRow().toString(); + + List> entries = + new ArrayList>(); + + // MDL 05 Jan 2014: Why is this equalsIgnoreCase?? + while (iterator.peek() != null && rowId.equalsIgnoreCase(iterator + .peek().getKey().getRow().toString())) { + entries.add(iterator.next()); + } + + AccumuloEdge edge = parser.parse(rowId, entries); + globals.getCaches().cache(edge, Edge.class); + + return edge; + } + }; + } + + public Iterable getEdges(String key, Object value) { + AccumuloGraphUtils.nullCheckProperty(key, value); + if (key.equalsIgnoreCase("label")) { + key = Constants.LABEL; + } + + BatchScanner scan = getBatchScanner(); + scan.fetchColumnFamily(new Text(key)); + + byte[] val = AccumuloByteSerializer.serialize(value); + if (val[0] != AccumuloByteSerializer.SERIALIZABLE) { + IteratorSetting is = new IteratorSetting(10, "filter", RegExFilter.class); + RegExFilter.setRegexs(is, null, null, null, Pattern.quote(new String(val)), false); + scan.addScanIterator(is); + + return new ScannerIterable(scan) { + + @Override + public Edge next(PeekingIterator> iterator) { + + Key k = iterator.next().getKey(); + + if (k.getColumnFamily().toString().equals(Constants.LABEL)) { + String[] vals = k.getColumnQualifier().toString().split(Constants.ID_DELIM); + return new AccumuloEdge(globals, k.getRow().toString(), + new AccumuloVertex(globals, vals[0]), + new AccumuloVertex(globals, vals[1]), null); + } + return new AccumuloEdge(globals, k.getRow().toString()); + } + }; + } else { + // TODO + throw new UnsupportedOperationException("Filtering on binary data not currently supported."); + } + } + + public void loadEndpointsAndLabel(AccumuloEdge edge) { + Scanner s = getScanner(); + + try { + s.setRange(new Range(edge.getId().toString())); + s.fetchColumnFamily(new Text(Constants.LABEL)); + Iterator> iter = s.iterator(); + if (!iter.hasNext()) { + dump(); + throw new AccumuloGraphException("Unable to find edge row: "+edge); + } + + Entry entry = iter.next(); + + String cq = entry.getKey().getColumnQualifier().toString(); + String[] ids = cq.split(Constants.ID_DELIM); + + String label = AccumuloByteSerializer.deserialize(entry.getValue().get()); + + edge.setVertices(new AccumuloVertex(globals, ids[0]), + new AccumuloVertex(globals, ids[1])); + edge.setLabel(label); + + } finally { + s.close(); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/core/ElementTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/core/ElementTableWrapper.java new file mode 100644 index 0000000..f922e33 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/core/ElementTableWrapper.java @@ -0,0 +1,226 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.core; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.util.StringFactory; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.mutator.property.ClearPropertyMutator; +import edu.jhuapl.tinkerpop.mutator.property.WritePropertyMutator; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.parser.PropertyParser; +import edu.jhuapl.tinkerpop.tables.BaseTableWrapper; + +/** + * Wrapper around tables with operations + * common to {@link Element}s. + */ +public abstract class ElementTableWrapper extends BaseTableWrapper { + + private BatchWriter writer; + + public ElementTableWrapper(GlobalInstances globals, String tableName) { + super(globals, tableName); + writer = super.getWriter(); + } + + /** + * Give a single instance of the writer for this table. + */ + @Override + protected BatchWriter getWriter() { + return writer; + } + + /** + * Read the given property from the backing table + * for the given element id. + * @param id + * @param key + * @return + */ + public V readProperty(Element element, String key) { + Scanner s = getScanner(); + + s.setRange(new Range(element.getId().toString())); + + Text colf = StringFactory.LABEL.equals(key) + ? new Text(Constants.LABEL) : new Text(key); + s.fetchColumnFamily(colf); + + V value = null; + + Iterator> iter = s.iterator(); + if (iter.hasNext()) { + value = AccumuloByteSerializer.deserialize(iter.next().getValue().get()); + } + s.close(); + + return value; + } + + /** + * Read all properties for the given element + * from the backing table. + * If the element has no properties, return an empty Map. + * If the element does not exist, return null. + * @param element + * @return + */ + public Map readAllProperties(Element element) { + return readProperties(element, null); + } + + /** + * Read the given properties for the given element. + * If propertyKeys is null, read all properties. + * If the element has no properties, return an empty Map. + * If the element does not exist, return null. + * @param id + * @param propertyKeys + * @return + */ + public Map readProperties(Element element, String[] propertyKeys) { + Scanner s = getScanner(); + s.setRange(Range.exact((String) element.getId())); + + // If propertyKeys is null, we read everything. + // Otherwise, limit to the given attributes. + if (propertyKeys != null) { + s.fetchColumnFamily(new Text(Constants.LABEL)); + + for (String key : propertyKeys) { + s.fetchColumnFamily(new Text(key)); + } + } + + Map props = new PropertyParser().parse(s); + s.close(); + + return props; + } + + /** + * Return true if the element with given id exists. + * @param id + * @return + */ + public boolean elementExists(String id) { + Scanner scan = null; + try { + scan = getScanner(); + scan.setRange(Range.exact(id)); + scan.fetchColumnFamily(new Text(Constants.LABEL)); + return new PropertyParser().parse(scan) != null; + + } finally { + if (scan != null) { + scan.close(); + } + } + } + + /** + * Get all property keys for the given element id. + * @param id + * @return + */ + public Set readPropertyKeys(Element element) { + Scanner s = getScanner(); + + s.setRange(new Range(element.getId().toString())); + + Set keys = new HashSet(); + + for (Entry entry : s) { + String cf = entry.getKey().getColumnFamily().toString(); + keys.add(cf); + } + + s.close(); + + // Remove some special keys. + keys.remove(Constants.IN_EDGE); + keys.remove(Constants.LABEL); + keys.remove(Constants.OUT_EDGE); + + return keys; + } + + /** + * Delete the property entry from property table. + * @param id + * @param key + */ + public void clearProperty(Element element, String key) { + Mutators.apply(getWriter(), + new ClearPropertyMutator(element.getId().toString(), key)); + globals.checkedFlush(); + } + + /** + * Write the given property to the property table. + * @param id + * @param key + * @param value + */ + public void writeProperty(Element element, String key, Object value) { + Mutators.apply(getWriter(), + new WritePropertyMutator(element.getId().toString(), + key, value)); + globals.checkedFlush(); + } + + /** + * Add custom iterator to the given scanner so that + * it will only return keys with value corresponding to an edge. + * @param scan + * @param labels + */ + protected void applyEdgeLabelValueFilter(Scanner scan, String... labels) { + StringBuilder regex = new StringBuilder(); + for (String lab : labels) { + if (regex.length() != 0) + regex.append("|"); + regex.append(".*"+Constants.ID_DELIM+"\\Q").append(lab).append("\\E$"); + } + + IteratorSetting is = new IteratorSetting(10, "edgeValueFilter", RegExFilter.class); + RegExFilter.setRegexs(is, null, null, null, regex.toString(), false); + scan.addScanIterator(is); + } + + public void close() { + // TODO? + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/core/VertexTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/core/VertexTableWrapper.java new file mode 100644 index 0000000..c607aa9 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/core/VertexTableWrapper.java @@ -0,0 +1,262 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.core; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map.Entry; +import java.util.regex.Pattern; + +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.core.util.PeekingIterator; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloEdge; +import edu.jhuapl.tinkerpop.AccumuloElement; +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.AccumuloGraphUtils; +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.Constants; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.ScannerIterable; +import edu.jhuapl.tinkerpop.mutator.vertex.AddVertexMutator; +import edu.jhuapl.tinkerpop.mutator.Mutator; +import edu.jhuapl.tinkerpop.mutator.Mutators; +import edu.jhuapl.tinkerpop.mutator.edge.EdgeEndpointsMutator; +import edu.jhuapl.tinkerpop.parser.VertexParser; + + +/** + * Wrapper around {@link Vertex} tables. + */ +public class VertexTableWrapper extends ElementTableWrapper { + + public VertexTableWrapper(GlobalInstances globals) { + super(globals, globals.getConfig().getVertexTableName()); + } + + /** + * Write a vertex with the given id. + * Note: This does not currently write the vertex's properties. + * @param vertex + */ + public void writeVertex(Vertex vertex) { + Mutators.apply(getWriter(), new AddVertexMutator(vertex.getId().toString())); + globals.checkedFlush(); + } + + /** + * Remove the given vertex. + * Note: This uses a BatchDeleter rather than {@link Mutator} + * because it is more efficient. + * @param vertex + */ + public void deleteVertex(Vertex vertex) { + BatchDeleter deleter = null; + + try { + deleter = getDeleter(); + deleter.setRanges(Arrays.asList(Range.exact((String) vertex.getId()))); + deleter.delete(); + + } catch (Exception e) { + throw new AccumuloGraphException(e); + } finally { + if (deleter != null) { + deleter.close(); + } + } + } + + /** + * Write edge endpoint information to the vertex table. + * @param edge + */ + public void writeEdgeEndpoints(Edge edge) { + Mutators.apply(getWriter(), new EdgeEndpointsMutator.Add(edge)); + globals.checkedFlush(); + } + + public void deleteEdgeEndpoints(Edge edge) { + Mutators.apply(getWriter(), new EdgeEndpointsMutator.Delete(edge)); + globals.checkedFlush(); + } + + public Iterable getEdges(Vertex vertex, Direction direction, + String... labels) { + Scanner scan = getScanner(); + scan.setRange(new Range(vertex.getId().toString())); + if (direction.equals(Direction.IN)) { + scan.fetchColumnFamily(new Text(Constants.IN_EDGE)); + } else if (direction.equals(Direction.OUT)) { + scan.fetchColumnFamily(new Text(Constants.OUT_EDGE)); + } else { + scan.fetchColumnFamily(new Text(Constants.IN_EDGE)); + scan.fetchColumnFamily(new Text(Constants.OUT_EDGE)); + } + + if (labels.length > 0) { + applyEdgeLabelValueFilter(scan, labels); + } + + return new ScannerIterable(scan) { + + @Override + public Edge next(PeekingIterator> iterator) { + // TODO better use of information readily available... + // TODO could also check local cache before creating a new + // instance? + + Entry kv = iterator.next(); + + String[] parts = kv.getKey().getColumnQualifier().toString().split(Constants.ID_DELIM); + String label = (new String(kv.getValue().get())).split(Constants.ID_DELIM)[1]; + + AccumuloEdge edge; + if (kv.getKey().getColumnFamily().toString().equalsIgnoreCase(Constants.IN_EDGE)) { + edge = new AccumuloEdge(globals, parts[1], + new AccumuloVertex(globals, kv.getKey().getRow().toString()), + new AccumuloVertex(globals, parts[0]), label); + } else { + edge = new AccumuloEdge(globals, parts[1], + new AccumuloVertex(globals, parts[0]), + new AccumuloVertex(globals, kv.getKey().getRow().toString()), label); + } + globals.getCaches().cache(edge, Edge.class); + + return edge; + } + }; + } + + public Iterable getVertices(Vertex vertex, Direction direction, String... labels) { + Scanner scan = getScanner(); + scan.setRange(new Range(vertex.getId().toString())); + if (direction.equals(Direction.IN)) { + scan.fetchColumnFamily(new Text(Constants.IN_EDGE)); + } else if (direction.equals(Direction.OUT)) { + scan.fetchColumnFamily(new Text(Constants.OUT_EDGE)); + } else { + scan.fetchColumnFamily(new Text(Constants.IN_EDGE)); + scan.fetchColumnFamily(new Text(Constants.OUT_EDGE)); + } + + if (labels != null && labels.length > 0) { + applyEdgeLabelValueFilter(scan, labels); + } + + return new ScannerIterable(scan) { + + @Override + public Vertex next(PeekingIterator> iterator) { + // TODO better use of information readily available... + // TODO could also check local cache before creating a new + // instance? + String[] parts = iterator.next().getKey().getColumnQualifier() + .toString().split(Constants.ID_DELIM); + + AccumuloVertex vertex = new AccumuloVertex(globals, parts[0]); + globals.getCaches().cache(vertex, Vertex.class); + + return vertex; + } + }; + } + + public Iterable getVertices() { + Scanner scan = getScanner(); + scan.fetchColumnFamily(new Text(Constants.LABEL)); + + if (globals.getConfig().getPreloadedProperties() != null) { + for (String key : globals.getConfig().getPreloadedProperties()) { + scan.fetchColumnFamily(new Text(key)); + } + } + + final VertexParser parser = new VertexParser(globals); + + return new ScannerIterable(scan) { + @Override + public Vertex next(PeekingIterator> iterator) { + // TODO could also check local cache before creating a new instance? + + String rowId = iterator.peek().getKey().getRow().toString(); + + List> entries = + new ArrayList>(); + + while (iterator.peek() != null && rowId.equals(iterator + .peek().getKey().getRow().toString())) { + entries.add(iterator.next()); + } + + AccumuloVertex vertex = parser.parse(rowId, entries); + globals.getCaches().cache(vertex, Vertex.class); + + return vertex; + } + }; + } + + public Iterable getVertices(String key, Object value) { + AccumuloGraphUtils.validateProperty(key, value); + + byte[] val = AccumuloByteSerializer.serialize(value); + if (val[0] != AccumuloByteSerializer.SERIALIZABLE) { + BatchScanner scan = getBatchScanner(); + scan.fetchColumnFamily(new Text(key)); + + IteratorSetting is = new IteratorSetting(10, "filter", RegExFilter.class); + RegExFilter.setRegexs(is, null, null, null, Pattern.quote(new String(val)), false); + scan.addScanIterator(is); + + return new ScannerIterable(scan) { + + @Override + public Vertex next(PeekingIterator> iterator) { + Entry kv = iterator.next(); + String key = kv.getKey().getColumnFamily().toString(); + Object value = AccumuloByteSerializer.deserialize(kv.getValue().get()); + + Vertex v = globals.getCaches().retrieve(kv.getKey().getRow().toString(), Vertex.class); + if (v == null) { + v = new AccumuloVertex(globals, kv.getKey().getRow().toString()); + } + + ((AccumuloElement) v).setPropertyInMemory(key, value); + globals.getCaches().cache(v, Vertex.class); + + return v; + } + }; + } else { + // TODO + throw new UnsupportedOperationException("Filtering on binary data not currently supported."); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/BaseKeyIndexTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/BaseKeyIndexTableWrapper.java new file mode 100644 index 0000000..42e9c1e --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/BaseKeyIndexTableWrapper.java @@ -0,0 +1,73 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.keyindex; + +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloGraphException; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.tables.BaseIndexValuesTableWrapper; +import edu.jhuapl.tinkerpop.tables.core.EdgeTableWrapper; +import edu.jhuapl.tinkerpop.tables.core.ElementTableWrapper; +import edu.jhuapl.tinkerpop.tables.core.VertexTableWrapper; + +/** + * Base class for key index tables. + */ +public class BaseKeyIndexTableWrapper extends BaseIndexValuesTableWrapper { + + protected BaseKeyIndexTableWrapper(GlobalInstances globals, + Class elementType, String tableName) { + super(globals, elementType, tableName); + } + + /** + * Rebuild this index for the given table. + * @param table + * @param key + */ + public void rebuildIndex(String key, Class elementClass) { + ElementTableWrapper wrapper = globals.getElementWrapper(elementClass); + if (wrapper instanceof VertexTableWrapper) { + for (Vertex v : ((VertexTableWrapper) wrapper).getVertices()) { + rebuild(wrapper, v, key); + } + } + else if (wrapper instanceof EdgeTableWrapper) { + for (Edge e : ((EdgeTableWrapper) wrapper).getEdges()) { + rebuild(wrapper, e, key); + } + } + else { + throw new AccumuloGraphException("Unexpected table wrapper: "+wrapper.getClass()); + } + globals.checkedFlush(); + } + + /** + * Add given element to index for the given key. + * @param element + * @param key + */ + private void rebuild(ElementTableWrapper wrapper, + Element element, String key) { + Object value = wrapper.readProperty(element, key); + if (value != null) { + setPropertyForIndex(element, key, value); + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/EdgeKeyIndexTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/EdgeKeyIndexTableWrapper.java new file mode 100644 index 0000000..4f9452d --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/EdgeKeyIndexTableWrapper.java @@ -0,0 +1,86 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.keyindex; + +import java.util.Arrays; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.PeekingIterator; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.Edge; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloEdge; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.ScannerIterable; +import edu.jhuapl.tinkerpop.parser.EdgeIndexParser; + +/** + * Wrapper around {@link Edge} index table. + */ +public class EdgeKeyIndexTableWrapper extends BaseKeyIndexTableWrapper { + + public EdgeKeyIndexTableWrapper(GlobalInstances globals) { + super(globals, Edge.class, globals.getConfig() + .getEdgeKeyIndexTableName()); + } + + /** + * Retrieve edges from the index table based + * on the given key/value. + * @param key + * @param value + * @return + */ + public Iterable getEdges(String key, Object value) { + Scanner s = getScanner(); + + Text row = new Text(AccumuloByteSerializer.serialize(value)); + s.setRange(Range.exact(row)); + s.fetchColumnFamily(new Text(key)); + + final EdgeIndexParser parser = new EdgeIndexParser(globals); + + return new ScannerIterable(s) { + + @Override + public Edge next(PeekingIterator> iterator) { + Entry entry = iterator.next(); + AccumuloEdge e = parser.parse(Arrays.asList(entry)); + + // Check if we have it cached already, in which + // case use the cached version. + AccumuloEdge cached = (AccumuloEdge) globals.getCaches() + .retrieve(e.getId(), Edge.class); + if (cached != null) { + for (String key : e.getPropertyKeysInMemory()) { + cached.setPropertyInMemory(key, e.getPropertyInMemory(key)); + } + + return cached; + } + + // We don't have it, so cache the new one and return it. + globals.getCaches().cache(e, Edge.class); + return e; + } + }; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/IndexedKeysListTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/IndexedKeysListTableWrapper.java new file mode 100644 index 0000000..612967b --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/IndexedKeysListTableWrapper.java @@ -0,0 +1,73 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.keyindex; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.accumulo.core.client.Scanner; + +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.util.ExceptionFactory; + +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.parser.IndexedItem; +import edu.jhuapl.tinkerpop.parser.IndexedItemsListParser; +import edu.jhuapl.tinkerpop.tables.BaseIndexedItemsListTableWrapper; + +/** + * Wraps the metadata tables which stores information + * about which property keys are indexed for different + * graph types. + */ +public class IndexedKeysListTableWrapper extends BaseIndexedItemsListTableWrapper { + + public IndexedKeysListTableWrapper(GlobalInstances globals) { + super(globals, globals.getConfig().getIndexedKeysTableName()); + } + + public void writeKeyMetadataEntry(String key, Class clazz) { + writeEntry(key, clazz); + } + + public void clearKeyMetadataEntry(String key, Class clazz) { + clearEntry(key, clazz); + } + + public Set getIndexedKeys(Class elementClass) { + if (elementClass == null) { + throw ExceptionFactory.classForElementCannotBeNull(); + } + + IndexedItemsListParser parser = new IndexedItemsListParser(elementClass); + + Scanner scan = null; + try { + scan = getScanner(); + + Set keys = new HashSet(); + for (IndexedItem item : parser.parse(scan)) { + keys.add(item.getKey()); + } + + return keys; + + } finally { + if (scan != null) { + scan.close(); + } + } + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/VertexKeyIndexTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/VertexKeyIndexTableWrapper.java new file mode 100644 index 0000000..b21b89f --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/keyindex/VertexKeyIndexTableWrapper.java @@ -0,0 +1,85 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.keyindex; + +import java.util.Arrays; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.util.PeekingIterator; +import org.apache.hadoop.io.Text; + +import com.tinkerpop.blueprints.Vertex; + +import edu.jhuapl.tinkerpop.AccumuloByteSerializer; +import edu.jhuapl.tinkerpop.AccumuloVertex; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.ScannerIterable; +import edu.jhuapl.tinkerpop.parser.VertexIndexParser; + +/** + * Wrapper around {@link Vertex} index table. + */ +public class VertexKeyIndexTableWrapper extends BaseKeyIndexTableWrapper { + + public VertexKeyIndexTableWrapper(GlobalInstances globals) { + super(globals, Vertex.class, globals.getConfig() + .getVertexKeyIndexTableName()); + } + + /** + * Use the index to retrieve vertices with the + * given key/value. + * @param key + * @param value + */ + public Iterable getVertices(String key, Object value) { + Scanner s = getScanner(); + + Text row = new Text(AccumuloByteSerializer.serialize(value)); + s.setRange(Range.exact(row)); + s.fetchColumnFamily(new Text(key)); + + final VertexIndexParser parser = new VertexIndexParser(globals); + + return new ScannerIterable(s) { + + @Override + public Vertex next(PeekingIterator> iterator) { + Entry entry = iterator.next(); + AccumuloVertex v = parser.parse(Arrays.asList(entry)); + + // Check if we have it cached already, in which + // case use the cached version. + AccumuloVertex cached = (AccumuloVertex) globals.getCaches() + .retrieve(v.getId(), Vertex.class); + if (cached != null) { + for (String key : v.getPropertyKeysInMemory()) { + cached.setPropertyInMemory(key, v.getPropertyInMemory(key)); + } + + return cached; + } + + // We don't have it, so cache the new one and return it. + globals.getCaches().cache(v, Vertex.class); + return v; + } + }; + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexListTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexListTableWrapper.java new file mode 100644 index 0000000..3d9c2c9 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexListTableWrapper.java @@ -0,0 +1,112 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.namedindex; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.accumulo.core.client.Scanner; + +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Index; +import com.tinkerpop.blueprints.util.ExceptionFactory; + +import edu.jhuapl.tinkerpop.AccumuloIndex; +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.parser.IndexedItem; +import edu.jhuapl.tinkerpop.parser.IndexedItemsListParser; +import edu.jhuapl.tinkerpop.tables.BaseIndexedItemsListTableWrapper; + +/** + * Wrapper around index metadata table. This lists + * names of indexes and their element types. + */ +public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper { + + public NamedIndexListTableWrapper(GlobalInstances globals) { + super(globals, globals.getConfig().getIndexNamesTableName()); + } + + public void writeIndexNameEntry(String indexName, + Class indexClass) { + writeEntry(indexName, indexClass); + } + + public void clearIndexNameEntry(String indexName, + Class indexClass) { + clearEntry(indexName, indexClass); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + public Iterable> getIndices() { + List> indexes = new ArrayList>(); + + IndexedItemsListParser parser = new IndexedItemsListParser(); + + Scanner scan = null; + try { + scan = getScanner(); + + for (IndexedItem item : parser.parse(scan)) { + indexes.add(new AccumuloIndex(globals, + item.getKey(), item.getElementClass())); + } + + return indexes; + + } finally { + if (scan != null) { + scan.close(); + } + } + } + + public Index getIndex(String indexName, + Class indexClass) { + IndexedItemsListParser parser = new IndexedItemsListParser(); + + Scanner scan = null; + try { + scan = getScanner(); + + for (IndexedItem item : parser.parse(scan)) { + if (item.getKey().equals(indexName)) { + if (item.getElementClass().equals(indexClass)) { + return new AccumuloIndex(globals, indexName, + indexClass); + } + else { + throw ExceptionFactory.indexDoesNotSupportClass(indexName, indexClass); + } + } + } + return null; + + } finally { + scan.close(); + } + } + + public Index createIndex(String indexName, Class indexClass) { + for (Index index : globals.getNamedIndexListWrapper().getIndices()) { + if (index.getIndexName().equals(indexName)) { + throw ExceptionFactory.indexAlreadyExists(indexName); + } + } + + writeIndexNameEntry(indexName, indexClass); + return new AccumuloIndex(globals, indexName, indexClass); + } +} diff --git a/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexTableWrapper.java b/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexTableWrapper.java new file mode 100644 index 0000000..dcb7585 --- /dev/null +++ b/src/main/java/edu/jhuapl/tinkerpop/tables/namedindex/NamedIndexTableWrapper.java @@ -0,0 +1,33 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop.tables.namedindex; + +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.IndexableGraph; + +import edu.jhuapl.tinkerpop.GlobalInstances; +import edu.jhuapl.tinkerpop.tables.BaseIndexValuesTableWrapper; + +/** + * Wrapper around a named index table (for {@link IndexableGraph}). + */ +public class NamedIndexTableWrapper extends BaseIndexValuesTableWrapper { + + public NamedIndexTableWrapper(GlobalInstances globals, + Class elementType, String indexName) { + super(globals, elementType, + globals.getConfig().getNamedIndexTableName(indexName)); + } +} diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..5ce79a4 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,6 @@ +log4j.rootLogger=INFO, stdout + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.Target=System.out +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c - %m%n diff --git a/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTest.java b/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTest.java index c06df4c..440ebb6 100644 --- a/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTest.java +++ b/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTest.java @@ -88,6 +88,7 @@ public class AccumuloGraphTest extends GraphTest { printTestPerformance("GraphSONReaderTestSuite", this.stopWatch()); } + @Override public void doTestSuite(final TestSuite testSuite) throws Exception { String doTest = System.getProperty("testTinkerGraph"); if (doTest == null || doTest.equals("true")) { @@ -112,12 +113,21 @@ public class AccumuloGraphTest extends GraphTest { new AccumuloGraphTest().generateGraph(); } + @Override public void dropGraph(final String graphDirectoryName) { if (graphDirectoryName != null) { - ((AccumuloGraph) generateGraph(graphDirectoryName)).clear(); + AccumuloGraphConfiguration cfg = AccumuloGraphTestUtils.generateGraphConfig(graphDirectoryName); + try { + for (String table : cfg.getConnector().tableOperations().list()) { + cfg.getConnector().tableOperations().delete(table); + } + } catch (Exception e) { + throw new AccumuloGraphException(e); + } } } + @Override public Object convertId(final Object id) { return id.toString(); } diff --git a/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTestUtils.java b/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTestUtils.java index 90ad4df..6cb8abd 100644 --- a/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTestUtils.java +++ b/src/test/java/edu/jhuapl/tinkerpop/AccumuloGraphTestUtils.java @@ -22,14 +22,11 @@ import edu.jhuapl.tinkerpop.AccumuloGraphConfiguration.InstanceType; public class AccumuloGraphTestUtils { public static AccumuloGraphConfiguration generateGraphConfig(String graphDirectoryName) { - AccumuloGraphConfiguration cfg = new AccumuloGraphConfiguration(); - cfg.setInstanceName("instanceName").setZooKeeperHosts("ZookeeperHostsString"); - cfg.setUser("root").setPassword(""); - cfg.setGraphName(graphDirectoryName).setCreate(true).setAutoFlush(true).setInstanceType(InstanceType.Mock); - return cfg; + return new AccumuloGraphConfiguration().setInstanceType(InstanceType.Mock) + .setGraphName(graphDirectoryName).setCreate(true); } public static Graph makeGraph(String name) { - return GraphFactory.open(generateGraphConfig(name).getConfiguration()); + return GraphFactory.open(generateGraphConfig(name)); } } diff --git a/src/test/java/edu/jhuapl/tinkerpop/AutoIndexTest.java b/src/test/java/edu/jhuapl/tinkerpop/AutoIndexTest.java index 8bb7127..1a7d2af 100644 --- a/src/test/java/edu/jhuapl/tinkerpop/AutoIndexTest.java +++ b/src/test/java/edu/jhuapl/tinkerpop/AutoIndexTest.java @@ -2,55 +2,117 @@ package edu.jhuapl.tinkerpop; import static org.junit.Assert.*; -import java.io.IOException; -import java.util.Map.Entry; - -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Scanner; -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; import org.junit.Test; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Element; import com.tinkerpop.blueprints.GraphFactory; import com.tinkerpop.blueprints.Vertex; public class AutoIndexTest { @Test - public void testIndexCreation() throws AccumuloException, AccumuloSecurityException, IOException, InterruptedException { - AccumuloGraph ag = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils.generateGraphConfig("AutoIndexTest").setAutoIndex(true).getConfiguration()); - String VERT = "1234"; - String KEY = "name"; - String VALUE = "bananaman"; + public void testVertexAutoIndex() throws Exception { + AccumuloGraph graph = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils + .generateGraphConfig("VertexAutoIndexTest").setAutoIndex(true).getConfiguration()); + String id = "1234"; + String key = "name"; + String value = "bananaman"; - Vertex v1 = ag.addVertex(VERT); - v1.setProperty(KEY, VALUE); + Vertex v1 = graph.addVertex(id); + v1.setProperty(key, value); - Scanner scan = ag.getVertexIndexScanner(); - for (Entry kv : scan) { - assertEquals(new String(AccumuloByteSerializer.serialize(VALUE)), kv.getKey().getRow().toString()); - assertEquals(KEY, kv.getKey().getColumnFamily().toString()); - assertEquals(VERT, kv.getKey().getColumnQualifier().toString()); + Iterable elements = graph.getGlobals() + .getVertexKeyIndexWrapper().readElementsFromIndex(key, value); + int count = 0; + for (Element element : elements) { + assertTrue(element instanceof Vertex); + assertEquals(id, element.getId()); + assertEquals(value, element.getProperty(key)); + count++; } + assertEquals(1, count); + graph.removeVertex(v1); + elements = graph.getGlobals() + .getVertexKeyIndexWrapper().readElementsFromIndex(key, value); + assertEquals(0, count(elements)); } @Test - public void testRegularCreation() throws AccumuloException, AccumuloSecurityException, IOException, InterruptedException { - AccumuloGraph ag = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils.generateGraphConfig("NoAutoIndexTest").getConfiguration()); - String VERT = "1234"; - String KEY = "name"; - String VALUE = "bananaman"; + public void testVertexNoAutoIndex() throws Exception { + AccumuloGraph graph = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils + .generateGraphConfig("VertexNoAutoIndexTest").getConfiguration()); + String id = "1234"; + String key = "name"; + String value = "bananaman"; - Vertex v1 = ag.addVertex(VERT); - v1.setProperty(KEY, VALUE); - - Scanner scan = ag.getVertexIndexScanner(); - for (Entry kv : scan) { - assertTrue(false); - } + Vertex v1 = graph.addVertex(id); + v1.setProperty(key, value); + Iterable elements = graph.getGlobals() + .getVertexKeyIndexWrapper().readElementsFromIndex(key, value); + assertEquals(0, count(elements)); } + @Test + public void testEdgeAutoIndex() throws Exception { + AccumuloGraph graph = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils + .generateGraphConfig("EdgeAutoIndex").setAutoIndex(true).getConfiguration()); + String id1 = "A"; + String id2 = "B"; + String eid = "X"; + String key = "name"; + String value = "bananaman"; + + Vertex v1 = graph.addVertex(id1); + Vertex v2 = graph.addVertex(id2); + Edge e = graph.addEdge(eid, v1, v2, "edge"); + e.setProperty(key, value); + + Iterable elements = graph.getGlobals() + .getEdgeKeyIndexWrapper().readElementsFromIndex(key, value); + int count = 0; + for (Element element : elements) { + assertTrue(element instanceof Edge); + assertEquals(eid, element.getId()); + assertEquals(value, element.getProperty(key)); + count++; + } + assertEquals(1, count); + + graph.removeVertex(v1); + elements = graph.getGlobals() + .getEdgeKeyIndexWrapper().readElementsFromIndex(key, value); + assertEquals(0, count(elements)); + } + + @Test + public void testEdgeNoAutoIndex() throws Exception { + AccumuloGraph graph = (AccumuloGraph) GraphFactory.open(AccumuloGraphTestUtils + .generateGraphConfig("EdgeNoAutoIndexTest").getConfiguration()); + String id1 = "A"; + String id2 = "B"; + String eid = "X"; + String key = "name"; + String value = "bananaman"; + + Vertex v1 = graph.addVertex(id1); + Vertex v2 = graph.addVertex(id2); + Edge e = graph.addEdge(eid, v1, v2, "edge"); + e.setProperty(key, value); + + Iterable elements = graph.getGlobals() + .getEdgeKeyIndexWrapper().readElementsFromIndex(key, value); + assertEquals(0, count(elements)); + } + + @SuppressWarnings("unused") + private static int count(Iterable it) { + int count = 0; + for (Object obj : it) { + count++; + } + return count; + } } diff --git a/src/test/java/edu/jhuapl/tinkerpop/ElementCacheTest.java b/src/test/java/edu/jhuapl/tinkerpop/ElementCacheTest.java index 9fbf06d..7c9f127 100644 --- a/src/test/java/edu/jhuapl/tinkerpop/ElementCacheTest.java +++ b/src/test/java/edu/jhuapl/tinkerpop/ElementCacheTest.java @@ -24,6 +24,8 @@ import com.tinkerpop.blueprints.Graph; import com.tinkerpop.blueprints.GraphFactory; import com.tinkerpop.blueprints.Vertex; +import edu.jhuapl.tinkerpop.cache.ElementCache; + public class ElementCacheTest { @Test diff --git a/src/test/java/edu/jhuapl/tinkerpop/ElementPropertyCachingTest.java b/src/test/java/edu/jhuapl/tinkerpop/ElementPropertyCachingTest.java new file mode 100644 index 0000000..f2a7b0b --- /dev/null +++ b/src/test/java/edu/jhuapl/tinkerpop/ElementPropertyCachingTest.java @@ -0,0 +1,232 @@ +/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.tinkerpop; + +import static org.junit.Assert.*; + +import org.junit.Test; + +import com.google.common.collect.Sets; +import com.tinkerpop.blueprints.Element; +import com.tinkerpop.blueprints.Graph; +import com.tinkerpop.blueprints.GraphFactory; + +/** + * Tests related to {@link Element}-based property caching. + */ +public class ElementPropertyCachingTest { + + private static final int TIMEOUT = 300000; + private static final String NON_CACHED = "noncached"; + private static final String CACHED = "cached"; + + @Test + public void testCachingDisabled() { + AccumuloGraphConfiguration cfg = + AccumuloGraphTestUtils.generateGraphConfig("cachingDisabled"); + assertTrue(cfg.getPropertyCacheTimeout(null) <= 0); + assertTrue(cfg.getPropertyCacheTimeout(NON_CACHED) <= 0); + assertTrue(cfg.getPropertyCacheTimeout(CACHED) <= 0); + + Graph graph = open(cfg); + load(graph); + + AccumuloVertex a = (AccumuloVertex) graph.getVertex("A"); + AccumuloVertex b = (AccumuloVertex) graph.getVertex("B"); + AccumuloVertex c = (AccumuloVertex) graph.getVertex("C"); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(true, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(true, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(null, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(null, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), c.getPropertyCache().keySet()); + + a.removeProperty(NON_CACHED); + b.removeProperty(NON_CACHED); + c.removeProperty(NON_CACHED); + a.removeProperty(CACHED); + b.removeProperty(CACHED); + c.removeProperty(CACHED); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(null, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(null, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(null, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(null, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), c.getPropertyCache().keySet()); + + graph.shutdown(); + } + + @Test + public void testSpecificCaching() { + AccumuloGraphConfiguration cfg = + AccumuloGraphTestUtils.generateGraphConfig("getProperty"); + cfg.setPropertyCacheTimeout(CACHED, TIMEOUT); + + assertTrue(cfg.getPropertyCacheTimeout(null) <= 0); + assertTrue(cfg.getPropertyCacheTimeout(NON_CACHED) <= 0); + assertEquals(TIMEOUT, cfg.getPropertyCacheTimeout(CACHED)); + + Graph graph = open(cfg); + load(graph); + + AccumuloVertex a = (AccumuloVertex) graph.getVertex("A"); + AccumuloVertex b = (AccumuloVertex) graph.getVertex("B"); + AccumuloVertex c = (AccumuloVertex) graph.getVertex("C"); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(true, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(true, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(null, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(true, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(CACHED), c.getPropertyCache().keySet()); + + a.removeProperty(NON_CACHED); + b.removeProperty(NON_CACHED); + c.removeProperty(NON_CACHED); + a.removeProperty(CACHED); + b.removeProperty(CACHED); + c.removeProperty(CACHED); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(null, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(null, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(null, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(null, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), c.getPropertyCache().keySet()); + + graph.shutdown(); + } + + @Test + public void testAllCaching() { + AccumuloGraphConfiguration cfg = + AccumuloGraphTestUtils.generateGraphConfig("setProperty"); + cfg.setPropertyCacheTimeout(null, TIMEOUT); + cfg.setPropertyCacheTimeout(CACHED, TIMEOUT); + + assertEquals(TIMEOUT, cfg.getPropertyCacheTimeout(null)); + assertEquals(TIMEOUT, cfg.getPropertyCacheTimeout(NON_CACHED)); + assertEquals(TIMEOUT, cfg.getPropertyCacheTimeout(CACHED)); + + Graph graph = open(cfg); + load(graph); + + AccumuloVertex a = (AccumuloVertex) graph.getVertex("A"); + AccumuloVertex b = (AccumuloVertex) graph.getVertex("B"); + AccumuloVertex c = (AccumuloVertex) graph.getVertex("C"); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(true, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(true, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(true, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(true, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(NON_CACHED), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(CACHED), c.getPropertyCache().keySet()); + + a.removeProperty(NON_CACHED); + b.removeProperty(NON_CACHED); + c.removeProperty(NON_CACHED); + a.removeProperty(CACHED); + b.removeProperty(CACHED); + c.removeProperty(CACHED); + + assertEquals(null, a.getProperty(NON_CACHED)); + assertEquals(null, b.getProperty(NON_CACHED)); + assertEquals(null, c.getProperty(NON_CACHED)); + assertEquals(null, a.getProperty(CACHED)); + assertEquals(null, b.getProperty(CACHED)); + assertEquals(null, c.getProperty(CACHED)); + + assertEquals(null, a.getPropertyCache().get(NON_CACHED)); + assertEquals(null, b.getPropertyCache().get(NON_CACHED)); + assertEquals(null, c.getPropertyCache().get(NON_CACHED)); + assertEquals(null, a.getPropertyCache().get(CACHED)); + assertEquals(null, b.getPropertyCache().get(CACHED)); + assertEquals(null, c.getPropertyCache().get(CACHED)); + + assertEquals(Sets.newHashSet(), a.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), b.getPropertyCache().keySet()); + assertEquals(Sets.newHashSet(), c.getPropertyCache().keySet()); + + graph.shutdown(); + } + + private static Graph open(AccumuloGraphConfiguration cfg) { + return GraphFactory.open(cfg); + } + + private static void load(Graph graph) { + graph.addVertex("A"); + graph.addVertex("B").setProperty(NON_CACHED, true); + graph.addVertex("C").setProperty(CACHED, true); + } +} diff --git a/src/test/java/edu/jhuapl/tinkerpop/PropertyCacheTest.java b/src/test/java/edu/jhuapl/tinkerpop/PropertyCacheTest.java index 6e86ff7..7cf106c 100644 --- a/src/test/java/edu/jhuapl/tinkerpop/PropertyCacheTest.java +++ b/src/test/java/edu/jhuapl/tinkerpop/PropertyCacheTest.java @@ -18,6 +18,8 @@ import static org.junit.Assert.*; import org.junit.Test; +import edu.jhuapl.tinkerpop.cache.PropertyCache; + /** * Test the {@link PropertyCache} object. */