Merge metadata tables into one table

This commit is contained in:
Michael Lieberman
2015-02-09 13:32:29 -05:00
parent bb6f9e80a7
commit 6faac76ba9
16 changed files with 128 additions and 185 deletions

View File

@@ -131,7 +131,7 @@ public abstract class AccumuloElement implements Element {
* @param element
*/
protected void removeElementFromNamedIndexes() {
for (Index<? extends Element> index : globals.getNamedIndexListWrapper().getIndices()) {
for (Index<? extends Element> index : globals.getIndexMetadataWrapper().getIndices()) {
((AccumuloIndex<? extends Element>) index).getWrapper().removeElementFromIndex(this);
}
}

View File

@@ -314,13 +314,13 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration");
}
for (Index<?> index : globals.getNamedIndexListWrapper().getIndices()) {
for (Index<?> index : globals.getIndexMetadataWrapper().getIndices()) {
if (index.getIndexName().equals(indexName)) {
throw ExceptionFactory.indexAlreadyExists(indexName);
}
}
return globals.getNamedIndexListWrapper().createIndex(indexName, indexClass);
return globals.getIndexMetadataWrapper().createIndex(indexName, indexClass);
}
@Override
@@ -332,7 +332,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration");
}
return globals.getNamedIndexListWrapper().getIndex(indexName, indexClass);
return globals.getIndexMetadataWrapper().getIndex(indexName, indexClass);
}
@Override
@@ -340,7 +340,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
if (globals.getConfig().getIndexableGraphDisabled()) {
throw new UnsupportedOperationException("IndexableGraph is disabled via the configuration");
}
return globals.getNamedIndexListWrapper().getIndices();
return globals.getIndexMetadataWrapper().getIndices();
}
@Override
@@ -350,7 +350,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
for (Index<? extends Element> index : getIndices()) {
if (index.getIndexName().equals(indexName)) {
globals.getNamedIndexListWrapper().clearIndexNameEntry(indexName, index.getIndexClass());
globals.getIndexMetadataWrapper().clearIndexNameEntry(indexName, index.getIndexClass());
try {
globals.getConfig().getConnector().tableOperations().delete(globals.getConfig()
@@ -373,7 +373,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
throw ExceptionFactory.classForElementCannotBeNull();
}
globals.getIndexedKeysListWrapper().clearKeyMetadataEntry(key, elementClass);
globals.getIndexMetadataWrapper().clearKeyMetadataEntry(key, elementClass);
String table = null;
if (elementClass.equals(Vertex.class)) {
@@ -406,7 +406,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
}
// Add key to indexed keys list.
globals.getIndexedKeysListWrapper().writeKeyMetadataEntry(key, elementClass);
globals.getIndexMetadataWrapper().writeKeyMetadataEntry(key, elementClass);
globals.checkedFlush();
// Reindex graph.
@@ -417,7 +417,7 @@ public class AccumuloGraph implements Graph, KeyIndexableGraph, IndexableGraph {
@Override
public <T extends Element> Set<String> getIndexedKeys(Class<T> elementClass) {
return globals.getIndexedKeysListWrapper().getIndexedKeys(elementClass);
return globals.getIndexMetadataWrapper().getIndexedKeys(elementClass);
}
/**

View File

@@ -973,26 +973,17 @@ implements Serializable {
}
/**
* Table listing the key-indexed properties
* of elements.
* Table containing index metadata.
* @return
*/
public String getIndexedKeysTableName() {
return getGraphName() + "_indexed_keys";
}
/**
* Table of existing named indexes.
* @return
*/
public String getIndexNamesTableName() {
return getGraphName() + "_index_names";
public String getIndexMetadataTableName() {
return getGraphName() + "_index_metadata";
}
List<String> getTableNames() {
return Arrays.asList(getVertexTableName(),
getEdgeTableName(), getVertexKeyIndexTableName(), getEdgeKeyIndexTableName(),
getIndexNamesTableName(), getIndexedKeysTableName());
getIndexMetadataTableName());
}
/**

View File

@@ -21,7 +21,7 @@ import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Index;
import com.tinkerpop.blueprints.IndexableGraph;
import edu.jhuapl.tinkerpop.tables.namedindex.NamedIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.index.NamedIndexTableWrapper;
/**
* Accumulo-based implementation for {@link IndexableGraph}.

View File

@@ -23,6 +23,9 @@ public class Constants {
private Constants() { }
/**
* Separate element ids in Accumulo entries.
*/
public static final String ID_DELIM = "__DELIM__";
public static final byte[] EMPTY = new byte[0];
@@ -31,7 +34,13 @@ public class Constants {
* Prefixes for various Accumulo entries.
*/
public static final String LABEL = "__LABEL__";
public static final String IN_EDGE = "__INEDGE__";
public static final String OUT_EDGE = "__OUTEDGE__";
public static final String IN_EDGE = "__IN_EDGE__";
public static final String OUT_EDGE = "__OUT_EDGE__";
public static final String EXISTS = "__EXISTS__";
/**
* Type of metadata to distinguish between
* entries in the metadata table.
*/
public static enum IndexMetadataEntryType {__INDEX_KEY__, __INDEX_NAME__};
}

View File

@@ -25,11 +25,10 @@ import edu.jhuapl.tinkerpop.cache.ElementCaches;
import edu.jhuapl.tinkerpop.tables.core.EdgeTableWrapper;
import edu.jhuapl.tinkerpop.tables.core.ElementTableWrapper;
import edu.jhuapl.tinkerpop.tables.core.VertexTableWrapper;
import edu.jhuapl.tinkerpop.tables.keyindex.BaseKeyIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.keyindex.EdgeKeyIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.keyindex.IndexedKeysListTableWrapper;
import edu.jhuapl.tinkerpop.tables.keyindex.VertexKeyIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.namedindex.NamedIndexListTableWrapper;
import edu.jhuapl.tinkerpop.tables.index.BaseKeyIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.index.EdgeKeyIndexTableWrapper;
import edu.jhuapl.tinkerpop.tables.index.IndexMetadataTableWrapper;
import edu.jhuapl.tinkerpop.tables.index.VertexKeyIndexTableWrapper;
/**
* Internal class gathering together instances of
@@ -72,12 +71,8 @@ public class GlobalInstances {
return new EdgeKeyIndexTableWrapper(this);
}
public IndexedKeysListTableWrapper getIndexedKeysListWrapper() {
return new IndexedKeysListTableWrapper(this);
}
public NamedIndexListTableWrapper getNamedIndexListWrapper() {
return new NamedIndexListTableWrapper(this);
public IndexMetadataTableWrapper getIndexMetadataWrapper() {
return new IndexMetadataTableWrapper(this);
}
public <T extends Element> ElementTableWrapper getElementWrapper(Class<T> clazz) {

View File

@@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
import com.tinkerpop.blueprints.Element;
import edu.jhuapl.tinkerpop.Constants;
import edu.jhuapl.tinkerpop.Constants.IndexMetadataEntryType;
import edu.jhuapl.tinkerpop.mutator.Mutator;
/**
@@ -32,18 +33,21 @@ public class IndexMetadataMutator {
public static class Add implements Mutator {
private final String key;
private final Class<? extends Element> clazz;
private final Class<? extends Element> elementClass;
private final IndexMetadataEntryType entryType;
public Add(String key, Class<? extends Element> clazz) {
public Add(String key, Class<? extends Element> elementClass,
IndexMetadataEntryType entryType) {
this.key = key;
this.clazz = clazz;
this.elementClass = elementClass;
this.entryType = entryType;
}
@Override
public Iterable<Mutation> create() {
Mutation m = new Mutation(key);
m.put(clazz.getName().getBytes(),
Constants.EMPTY, Constants.EMPTY);
m.put(entryType.name().getBytes(),
elementClass.getName().getBytes(), Constants.EMPTY);
return Lists.newArrayList(m);
}
}
@@ -51,17 +55,21 @@ public class IndexMetadataMutator {
public static class Delete implements Mutator {
private final String key;
private final Class<? extends Element> clazz;
private final Class<? extends Element> elementClass;
private final IndexMetadataEntryType entryType;
public Delete(String indexName, Class<? extends Element> clazz) {
public Delete(String indexName, Class<? extends Element> elementClass,
IndexMetadataEntryType entryType) {
this.key = indexName;
this.clazz = clazz;
this.elementClass = elementClass;
this.entryType = entryType;
}
@Override
public Iterable<Mutation> create() {
Mutation m = new Mutation(key);
m.putDelete(clazz.getName().getBytes(), Constants.EMPTY);
m.putDelete(entryType.name().getBytes(),
elementClass.getName().getBytes());
return Lists.newArrayList(m);
}
}

View File

@@ -73,7 +73,7 @@ public class IndexedItemsListParser implements EntryParser<List<IndexedItem>> {
Class<? extends Element> clazz;
try {
clazz = (Class<? extends Element>) Class.forName(entry.getKey()
.getColumnFamily().toString());
.getColumnQualifier().toString());
} catch (ClassNotFoundException e) {
throw new AccumuloGraphException(e);
}

View File

@@ -1,42 +0,0 @@
/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables;
import com.tinkerpop.blueprints.Element;
import edu.jhuapl.tinkerpop.GlobalInstances;
import edu.jhuapl.tinkerpop.mutator.Mutators;
import edu.jhuapl.tinkerpop.mutator.index.IndexMetadataMutator;
/**
* Wraps the metadata tables which stores information
* about which property keys are indexed for different
* graph types.
*/
public abstract class BaseIndexedItemsListTableWrapper extends BaseTableWrapper {
protected BaseIndexedItemsListTableWrapper(GlobalInstances globals,
String tableName) {
super(globals, tableName);
}
protected void writeEntry(String key, Class<? extends Element> clazz) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Add(key, clazz));
}
protected void clearEntry(String key, Class<? extends Element> clazz) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Delete(key, clazz));
}
}

View File

@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables;
package edu.jhuapl.tinkerpop.tables.index;
import java.util.Arrays;
import java.util.Collections;
@@ -46,6 +46,7 @@ import edu.jhuapl.tinkerpop.mutator.index.IndexValueMutator;
import edu.jhuapl.tinkerpop.parser.EdgeIndexParser;
import edu.jhuapl.tinkerpop.parser.ElementIndexParser;
import edu.jhuapl.tinkerpop.parser.VertexIndexParser;
import edu.jhuapl.tinkerpop.tables.BaseTableWrapper;
/**
* Wrapper around index tables containing properties
@@ -98,7 +99,7 @@ public abstract class BaseIndexValuesTableWrapper extends BaseTableWrapper {
boolean force) {
AccumuloGraphUtils.validateProperty(key, value);
if (force || globals.getConfig().getAutoIndex() ||
globals.getIndexedKeysListWrapper()
globals.getIndexMetadataWrapper()
.getIndexedKeys(elementType).contains(key)) {
BatchWriter writer = getWriter();

View File

@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.keyindex;
package edu.jhuapl.tinkerpop.tables.index;
import com.tinkerpop.blueprints.CloseableIterable;
import com.tinkerpop.blueprints.Edge;
@@ -21,7 +21,6 @@ import com.tinkerpop.blueprints.Vertex;
import edu.jhuapl.tinkerpop.AccumuloGraphException;
import edu.jhuapl.tinkerpop.GlobalInstances;
import edu.jhuapl.tinkerpop.tables.BaseIndexValuesTableWrapper;
import edu.jhuapl.tinkerpop.tables.core.EdgeTableWrapper;
import edu.jhuapl.tinkerpop.tables.core.ElementTableWrapper;
import edu.jhuapl.tinkerpop.tables.core.VertexTableWrapper;
@@ -29,7 +28,7 @@ import edu.jhuapl.tinkerpop.tables.core.VertexTableWrapper;
/**
* Base class for key index tables.
*/
public class BaseKeyIndexTableWrapper extends BaseIndexValuesTableWrapper {
public abstract class BaseKeyIndexTableWrapper extends BaseIndexValuesTableWrapper {
protected BaseKeyIndexTableWrapper(GlobalInstances globals,
Class<? extends Element> elementType, String tableName) {

View File

@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.keyindex;
package edu.jhuapl.tinkerpop.tables.index;
import java.util.Arrays;
import java.util.Map.Entry;

View File

@@ -12,43 +12,84 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.namedindex;
package edu.jhuapl.tinkerpop.tables.index;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.apache.accumulo.core.client.Scanner;
import org.apache.hadoop.io.Text;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.Index;
import com.tinkerpop.blueprints.IndexableGraph;
import com.tinkerpop.blueprints.KeyIndexableGraph;
import com.tinkerpop.blueprints.util.ExceptionFactory;
import edu.jhuapl.tinkerpop.AccumuloIndex;
import edu.jhuapl.tinkerpop.Constants.IndexMetadataEntryType;
import edu.jhuapl.tinkerpop.GlobalInstances;
import edu.jhuapl.tinkerpop.mutator.Mutators;
import edu.jhuapl.tinkerpop.mutator.index.IndexMetadataMutator;
import edu.jhuapl.tinkerpop.parser.IndexedItem;
import edu.jhuapl.tinkerpop.parser.IndexedItemsListParser;
import edu.jhuapl.tinkerpop.tables.BaseIndexedItemsListTableWrapper;
import edu.jhuapl.tinkerpop.tables.BaseTableWrapper;
/**
* Wrapper around index metadata table. This lists
* names of indexes and their element types.
* Stores metadata, in particular the indexed keys
* for {@link KeyIndexableGraph}, and the list of
* named indexes for {@link IndexableGraph}.
*/
public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper {
public class IndexMetadataTableWrapper extends BaseTableWrapper {
public NamedIndexListTableWrapper(GlobalInstances globals) {
super(globals, globals.getConfig().getIndexNamesTableName());
public IndexMetadataTableWrapper(GlobalInstances globals) {
super(globals, globals.getConfig().getIndexMetadataTableName());
}
public void writeIndexNameEntry(String indexName,
Class<? extends Element> indexClass) {
writeEntry(indexName, indexClass);
//////// Methods for KeyIndexableGraph ////////
public void writeKeyMetadataEntry(String key, Class<? extends Element> clazz) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Add(key, clazz,
IndexMetadataEntryType.__INDEX_KEY__));
}
public void clearIndexNameEntry(String indexName,
Class<? extends Element> indexClass) {
clearEntry(indexName, indexClass);
public void clearKeyMetadataEntry(String key, Class<? extends Element> clazz) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Delete(key, clazz,
IndexMetadataEntryType.__INDEX_KEY__));
}
public <T extends Element> Set<String> getIndexedKeys(Class<T> elementClass) {
if (elementClass == null) {
throw ExceptionFactory.classForElementCannotBeNull();
}
IndexedItemsListParser parser = new IndexedItemsListParser(elementClass);
Scanner scan = null;
try {
scan = getScanner();
scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_KEY__.name()));
Set<String> keys = new HashSet<String>();
for (IndexedItem item : parser.parse(scan)) {
keys.add(item.getKey());
}
return keys;
} finally {
if (scan != null) {
scan.close();
}
}
}
//////// Methods for IndexableGraph ////////
@SuppressWarnings({"rawtypes", "unchecked"})
public Iterable<Index<? extends Element>> getIndices() {
List<Index<? extends Element>> indexes = new ArrayList<Index<? extends Element>>();
@@ -58,6 +99,7 @@ public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper
Scanner scan = null;
try {
scan = getScanner();
scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_NAME__.name()));
for (IndexedItem item : parser.parse(scan)) {
indexes.add(new AccumuloIndex(globals,
@@ -80,6 +122,7 @@ public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper
Scanner scan = null;
try {
scan = getScanner();
scan.fetchColumnFamily(new Text(IndexMetadataEntryType.__INDEX_NAME__.name()));
for (IndexedItem item : parser.parse(scan)) {
if (item.getKey().equals(indexName)) {
@@ -101,8 +144,9 @@ public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper
}
}
public <T extends Element> Index<T> createIndex(String indexName, Class<T> indexClass) {
for (Index<?> index : globals.getNamedIndexListWrapper().getIndices()) {
public <T extends Element> Index<T> createIndex(String indexName,
Class<T> indexClass) {
for (Index<?> index : getIndices()) {
if (index.getIndexName().equals(indexName)) {
throw ExceptionFactory.indexAlreadyExists(indexName);
}
@@ -111,4 +155,16 @@ public class NamedIndexListTableWrapper extends BaseIndexedItemsListTableWrapper
writeIndexNameEntry(indexName, indexClass);
return new AccumuloIndex<T>(globals, indexName, indexClass);
}
private void writeIndexNameEntry(String indexName,
Class<? extends Element> indexClass) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Add(indexName,
indexClass, IndexMetadataEntryType.__INDEX_NAME__));
}
public void clearIndexNameEntry(String indexName,
Class<? extends Element> indexClass) {
Mutators.apply(getWriter(), new IndexMetadataMutator.Delete(indexName,
indexClass, IndexMetadataEntryType.__INDEX_NAME__));
}
}

View File

@@ -12,13 +12,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.namedindex;
package edu.jhuapl.tinkerpop.tables.index;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.IndexableGraph;
import edu.jhuapl.tinkerpop.GlobalInstances;
import edu.jhuapl.tinkerpop.tables.BaseIndexValuesTableWrapper;
/**
* Wrapper around a named index table (for {@link IndexableGraph}).

View File

@@ -12,7 +12,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.keyindex;
package edu.jhuapl.tinkerpop.tables.index;
import java.util.Arrays;
import java.util.Map.Entry;

View File

@@ -1,73 +0,0 @@
/* Copyright 2014 The Johns Hopkins University Applied Physics Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package edu.jhuapl.tinkerpop.tables.keyindex;
import java.util.HashSet;
import java.util.Set;
import org.apache.accumulo.core.client.Scanner;
import com.tinkerpop.blueprints.Element;
import com.tinkerpop.blueprints.util.ExceptionFactory;
import edu.jhuapl.tinkerpop.GlobalInstances;
import edu.jhuapl.tinkerpop.parser.IndexedItem;
import edu.jhuapl.tinkerpop.parser.IndexedItemsListParser;
import edu.jhuapl.tinkerpop.tables.BaseIndexedItemsListTableWrapper;
/**
* Wraps the metadata tables which stores information
* about which property keys are indexed for different
* graph types.
*/
public class IndexedKeysListTableWrapper extends BaseIndexedItemsListTableWrapper {
public IndexedKeysListTableWrapper(GlobalInstances globals) {
super(globals, globals.getConfig().getIndexedKeysTableName());
}
public void writeKeyMetadataEntry(String key, Class<? extends Element> clazz) {
writeEntry(key, clazz);
}
public void clearKeyMetadataEntry(String key, Class<? extends Element> clazz) {
clearEntry(key, clazz);
}
public <T extends Element> Set<String> getIndexedKeys(Class<T> elementClass) {
if (elementClass == null) {
throw ExceptionFactory.classForElementCannotBeNull();
}
IndexedItemsListParser parser = new IndexedItemsListParser(elementClass);
Scanner scan = null;
try {
scan = getScanner();
Set<String> keys = new HashSet<String>();
for (IndexedItem item : parser.parse(scan)) {
keys.add(item.getKey());
}
return keys;
} finally {
if (scan != null) {
scan.close();
}
}
}
}