commit 365b11fd6b64c60d6fda10760eded37d62ca486b Author: Dennis Patrone Date: Wed Sep 2 15:42:52 2015 -0400 initial import diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9991357 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +* text eol=lf + +*.java text +*.txt text + +*.png binary diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8f71f43 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..14b2efd --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +ProxyInstance +============= + +This is an implementation of the Accumulo `Instance` Application Programming Interface (API) which uses the Accumulo-supplied, Apache Thrift-based +proxy server as the back-end for communicating with an Accumulo instance. + +For more information, see the [Users' Manual](http://jhuapl.github.io/accumulo-proxy-instance/proxy_instance_user_manual.html). diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..3b0a6b8 --- /dev/null +++ b/pom.xml @@ -0,0 +1,657 @@ + + + 4.0.0 + edu.jhuapl.accumulo + proxy-instance-project + 1.0.0-SNAPSHOT + pom + Accumulo Proxy Instance Project + Provides an Accumulo Java API Instance implementation that communicates via the Accumulo Thrift-based Proxy server. + https://github.com/JHUAPL/accumulo-proxy-instance + + JHU/APL + http://www.jhuapl./edu/ + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + Dennis Patrone + https://github.com/dennispatrone + + + David Patrone + + + + proxy-instance + proxy-instance-docs + + + scm:git:git@github.com/JHUAPL/accumulo-proxy-instance.git + scm:git:git@github.com:JHUAPL/accumulo-proxy-instance.git + git@github.com:JHUAPL/accumulo-proxy-instance.git + + + + ossrh + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + + 1.6.2 + 3.0.1 + 1 + 1.7 + 1.7 + 3.0.4 + UTF-8 + false + + + + + junit + junit + 4.12 + + + org.slf4j + slf4j-api + 1.7.12 + + + org.slf4j + slf4j-log4j12 + 1.7.12 + + + org.slf4j + slf4j-nop + 1.7.12 + + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + ${findbugs.version} + + true + Max + true + true + 16 + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.15 + + + com.github.ekryd.sortpom + sortpom-maven-plugin + 2.4.0 + + recommended_2008_06 + false + \n + false + 2 + scope,groupId,artifactId + true + Stop + + + + org.apache.maven.plugins + maven-javadoc-plugin + + ${project.reporting.outputEncoding} + true + docs + ${maven.compiler.target} + -J-Xmx512m + + + + org.apache.maven.plugins + maven-site-plugin + + true + + + + org.apache.maven.plugins + maven-failsafe-plugin + 2.18.1 + + + integration-test + + integration-test + + + + verify + + verify + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + -Xmx1G + + + + org.asciidoctor + asciidoctor-maven-plugin + 1.5.2.1 + + + org.codehaus.mojo + cobertura-maven-plugin + 2.7 + + true + + xml + html + + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + org.apache.maven.plugins + maven-plugin-plugin + [3.2,) + + helpmojo + descriptor + + + + + + + + + com.googlecode.maven-java-formatter-plugin + maven-java-formatter-plugin + [0.4,) + + format + verify + + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + [2.13,) + + check + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + [1.0,) + + enforce + + + + + + + + + com.github.ekryd.sortpom + sortpom-maven-plugin + [2.4.0,) + + sort + verify + + + + + + + + + com.mycila + license-maven-plugin + [2.11,) + + format + check + + + + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 1.4 + + + + [${maven.min-version},) + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-mvn + + enforce + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5 + + true + false + release + deploy + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.3 + true + + ossrh + https://oss.sonatype.org/ + true + + + + com.github.ekryd.sortpom + sortpom-maven-plugin + + + sort-pom + + sort + + process-sources + + + verify-sorted-pom + + verify + + process-resources + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + warning + true + + + + com.puppycrawl.tools + checkstyle + 6.3 + + + + + check-style + + check + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + org.codehaus.mojo + findbugs-maven-plugin + + + run-findbugs + + check + + + + + + org.apache.maven.plugins + maven-scm-publish-plugin + 1.1 + + + scm-publish + + publish-scm + + site-deploy + + + + + + + org.apache.maven.wagon + wagon-ssh + 2.8 + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + + javadoc + + + + + + org.apache.maven.plugins + maven-pmd-plugin + 3.5 + + html + true + ${maven.compiler.target} + + + + org.apache.maven.plugins + maven-project-info-reports-plugin + 2.8 + + false + false + + + + + summary + index + dependencies + issue-tracking + scm + + + + + + org.apache.maven.plugins + maven-jxr-plugin + 2.5 + + + org.codehaus.mojo + findbugs-maven-plugin + ${findbugs.version} + + true + true + true + Max + Medium + false + + + + + + + release + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + attach-javadocs + + jar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.5 + + + sign-artifacts + + sign + + verify + + + + + + + + javadoc + + + + org.apache.maven.plugins + maven-source-plugin + 2.2.1 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + attach-javadocs + + jar + + + + + + + + + fatjar + + + + org.apache.maven.plugins + maven-shade-plugin + 2.3 + + + + shade + + package + + + + + + + + diff --git a/proxy-instance-docs/pom.xml b/proxy-instance-docs/pom.xml new file mode 100644 index 0000000..61a4808 --- /dev/null +++ b/proxy-instance-docs/pom.xml @@ -0,0 +1,111 @@ + + + 4.0.0 + + edu.jhuapl.accumulo + proxy-instance-project + 1.0.0-SNAPSHOT + + proxy-instance-docs + pom + Documentation + User documentation for Accumulo ProxyInstance. + + + edu.jhuapl.accumulo + proxy-instance + ${project.version} + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-asciidoc + + copy-resources + + compile + + ${project.build.directory}/asciidoc + + + src/main/asciidoc + + + + + + + + org.asciidoctor + asciidoctor-maven-plugin + + html + book + true + ${project.build.directory}/asciidoc/images + ${project.build.directory}/asciidoc + highlightjs + + ${project.version} + ${accumulo.version} + + + + + output-html + + process-asciidoc + + prepare-package + + + + + org.codehaus.mojo + exec-maven-plugin + + + prep-output-dir + + exec + + compile + + mkdir + + -p + ${project.build.directory}/asciidoc + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-user-manual-html + + attach-artifact + + + + + ${project.build.directory}/generated-docs/proxy_instance_user_manual.html + html + user-manual + + + + + + + + + diff --git a/proxy-instance-docs/pom.xml.releaseBackup b/proxy-instance-docs/pom.xml.releaseBackup new file mode 100644 index 0000000..9191128 --- /dev/null +++ b/proxy-instance-docs/pom.xml.releaseBackup @@ -0,0 +1,112 @@ + + + 4.0.0 + + edu.jhuapl.accumulo + proxy-instance-project + 1.0.0-SNAPSHOT + .. + + proxy-instance-docs + pom + Documentation + User documentation for Accumulo ProxyInstance. + + + edu.jhuapl.accumulo + proxy-instance + ${project.version} + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-asciidoc + + copy-resources + + compile + + ${project.build.directory}/asciidoc + + + src/main/asciidoc + + + + + + + + org.asciidoctor + asciidoctor-maven-plugin + + html + book + true + ${project.build.directory}/asciidoc/images + ${project.build.directory}/asciidoc + highlightjs + + ${project.version} + ${accumulo.version} + + + + + output-html + + process-asciidoc + + prepare-package + + + + + org.codehaus.mojo + exec-maven-plugin + + + prep-output-dir + + exec + + compile + + mkdir + + -p + ${project.build.directory}/asciidoc + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + attach-user-manual-html + + attach-artifact + + + + + ${project.build.directory}/generated-docs/proxy_instance_user_manual.html + html + user-manual + + + + + + + + + diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/building.txt b/proxy-instance-docs/src/main/asciidoc/chapters/building.txt new file mode 100644 index 0000000..192a9af --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/building.txt @@ -0,0 +1,16 @@ +== Building + +The source hierarchy for `ProxyInstance` is: + +* *proxy-instance-project*: POM contains general plugin versions and configurations +** *proxy-instance*: The actual source and test code for the `ProxyInstance` +** *proxy-instance-docs*: This documentation +** *proxy-instance-build*: Resources necessary for building (e.g, license information, formatting templates) + +To build the system, you can execute: + + mvn package + +To build it while executing the integration tests (requires an external Accumulo instance and Proxy Server configured and running): + + mvn failsafe:integration-test package -Daccumulo.proxy.host=myhost -Daccumulo.proxy.port=myport -Daccumulo.proxy.user=myuser -Daccumulo.proxy.password=mypassword diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/copyright.txt b/proxy-instance-docs/src/main/asciidoc/chapters/copyright.txt new file mode 100644 index 0000000..99d049e --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/copyright.txt @@ -0,0 +1,13 @@ +Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/development.txt b/proxy-instance-docs/src/main/asciidoc/chapters/development.txt new file mode 100644 index 0000000..8a9fad4 --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/development.txt @@ -0,0 +1,64 @@ +== Development + +This chapter includes information for those wishing to further development on the `ProxyInstance`. + +=== Unit and Integration Tests + +The `ProxyInstance` repository contains a number of unit tests (`XxxTest.java`) and integration tests (`XxxIT.java`) in +the `src/test` directory. But more are needed. + +As a convenience, developers may choose to have their unit and integration test classes subclass the provided `ConnectorBase` +class. The ConnectorBase handles obtaining an `Instance` and `Connector` references that can be utilized by the +tests. + +A special tag interface called `IntegrationTest` has also been created. If a sub-class of ConnectorBase containing tests +implements the `IntegrationTest` interface, it will be provided a `ProxyInstance` configured using the required `System` +properties: `accumulo.proxy.host`, `accumulo.proxy.port`, `accumulo.proxy.user`, and +`accumulo.proxy.password`. The integration tests assume a real Proxy Server is running somewhere on the network. If the +test class does not implement `IntegrationTest`, a local Proxy Server will be created backed by a `MockInstance` +Accumulo (we do not use the mini due to https://issues.apache.org/jira/browse/ACCUMULO-3293[ACCUMULO-3293]). + +A common pattern when testing is to be able to test the logic via the local mock instance proxy server as a unit test and +test the same capabilities again against an external proxy server as an integration test. The current infrastructure +enables this without duplication of code. The general pattern is: + +. Create your unit tests by subclassing `ConnectorBase` and writing your tests against the `protected` member variables +`instance` and `connector`. When run as a unit tests, these will automatically be connected to a local, mock-based +proxy server. ++ +[source,java] +---- +public class SomeClassTest extends ConnectorBase { + + @Test + public void testSomething() { + // make use of member variables 'instance' and 'connector' as needed, + // already initialized by parent ConnectorBase + BatchWriter bw = connector.createBatchWriter(...); + + // do more stuff... + } +} +---- + +. Create your integration test by simply sub-classing your unit test and implement the tag interface `IntegrationTest`. +That is it. When you run the integration tests, the `instance` and `connector` member variables will be connected to +the remote proxy server based on the required system parameters. ++ +[source,java] +---- +public class SomeClassIT extends SomeClassTest implements IntegrationTest { + + // Do nothing more; the ConnectorBase parent will recoginize this is an + // IntegrationTest and provide the appropriate 'instance' and + // 'connector' references and then execute all of the same logic the unit + // test executed. + + // Of course, you can create new methods here if there is additional + // integration testing desired beyond what the unit test performed. + +} +---- + + + diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/introduction.txt b/proxy-instance-docs/src/main/asciidoc/chapters/introduction.txt new file mode 100644 index 0000000..eb14af1 --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/introduction.txt @@ -0,0 +1,32 @@ +== Introduction + + +This is an implementation of the Accumulo `Instance` Application Programming Interface (API) which uses the Accumulo-supplied, Apache Thrift-based +proxy server as the back-end for communicating with an Accumulo instance. + +Apache Accumulo provides two methods for interacting with an Accumulo instance: a Java-based client library and an Apache Thrift-based +API. The Java-based client library requires that the client application have direct network access to all data nodes in the Accumulo instance +cloud as the client application communicates directly with the tablet servers where the data is stored. + +image::traditional.png["Traditional",width="800",align="center"] + +The Thrift-based API interacts with an Accumulo instance through a Proxy Server; the client application only needs direct network +access to the Proxy Server (the proxy server, in turn, communicates with the tablet servers directly on behalf of the requesting +client application). While providing similar capabilities as the Java-based client library, the Thrift-based API is significantly +different than the Java-based API. The Thrift API was originally developed to provide Accumulo access to non-Java applications. However, +in situations where the Accumulo cloud is not entirely network addressable by Java-based client applications (e.g., isolated behind a firewall), +it is useful to allow Java clients to utilize the proxy service. Furthermore, It would be ideal to expose the proxy service through the same API +as the traditional Java-based client library to protect client source code from significant changes based only on differences in the +network topology. + +This Proxy `Instance` implementation provides such an implementation. It is a Java-based client library for interacting with Accumulo's +ProxyService via the Thrift interface, but exposes the same Java API as the traditional Java-based client library. This enables, +in the future (e.g., after development and testing) by moving the client code onto the isolated Accumulo network and with a simple switch +of the `Instance` type created, the Java client application can take advantage of the performance increase using the traditional Java +client library. + + +image::proxy.png["Proxy",width="800",align="center"] + + +This version was written, compiled, and tested against Accumulo {accumuloVersion}. \ No newline at end of file diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/known_issues.txt b/proxy-instance-docs/src/main/asciidoc/chapters/known_issues.txt new file mode 100644 index 0000000..cdde789 --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/known_issues.txt @@ -0,0 +1,10 @@ +== Known Issues + +Here we document the known issues. + +* Not all Java APIs are supported through the Proxy interface. Currently, the proxy throws an `UnsupportedOperationException` in +such cases. We should enter tickets to update the proxy thrift interface to support these additional capabilities. +* We currently need to close the instance which is not part of the public API. Maybe we should open and close the transport every +time we access the instance so we can open/close the transport each time? Is there a better way to handle closing the transport +without incurring the overhead of re-establishing the connection on every call? +* Others? diff --git a/proxy-instance-docs/src/main/asciidoc/chapters/usage.txt b/proxy-instance-docs/src/main/asciidoc/chapters/usage.txt new file mode 100644 index 0000000..d323c1b --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/chapters/usage.txt @@ -0,0 +1,75 @@ +== Usage + +This section contains a brief introduction to getting setup using the Proxy Instance. + +. You must have the Accumulo Proxy Server up and running. See http://accumulo.apache.org/1.6/accumulo_user_manual.html#_proxy for more information. + +. Include this for maven (or download the latest JARs from http://search.maven.org/[Maven Central]) ++ +[source,xml,indent=0] +---- + + edu.jhuapl.accumulo + proxy-instance + ${proxy.version} + +---- ++ +The current version is {docVersion}. + +. Create an instance of `ProxyInstance` and provide it the hostname (or IP address) and port where the proxy is running. ++ +[source,java] +---- + Instance instance = new ProxyInstance("proxyhost", 4567); + Connector connector = instance.getConnector(user, new PassworkToken(password)); +---- + +. Use the `instance` and `connector` objects as you normally would using the traditional Java API. + +. When finished, we also need to close the `ProxyInstance`. Unfortunately, a method to close an `Instance` does not exist in the public +API. Therefore, we must add something like this when we are done with the `Instance`: ++ +[source,java] +---- + if (instance instanceof ProxyInstance) { + ((ProxyInstance)instance).close(); + } + +---- + +=== Configuration Parameters + +==== BatchScanner Fetch Size + +When fetching data from a `BatchScanner,` the Proxy Server API allows you to request data in batches. By batching multiple {key,value} +pairs up into a single fetch request, data latency and network bandwidth consumed may be reduced. The pairs are queued in the Proxy Server +and returned in a batch when the fetch size (or the end of the `Scanner`) is reached. + +The `ProxyInstance` `BatchScanner` implementation defaults to a fetch size of 1,000 pairs. In general, this is good for fetching a lot of +"small" data from very large tables. However, in certain circumstances this batching can actually increase the data latency for the first +data element. The ProxyServer will fill the entire fetch size buffer before sending any data to the client. If very selective filtering is +applied to the scanner on the server-side, it may take a long time for the Proxy Server to find 1,000 pairs to return even if a few are found +very quickly. Furthermore, if the keys and/or values are sufficiently large (or the RAM available to the Proxy Server is sufficiently +limited), queuing 1,000 pairs in RAM can cause the Proxy Server to crash with an `OutOfMemoryException`. + +Therefore, the `ProxyInstance` provides a way for clients to modify the default fetch size via an optional argument to the `ProxyInstance` +constructor. Unfortunately, there is no place within the Java API to specify a fetch size for a `BatchScanner` on a per-scanner +basis. Therefore, changing the fetch size via the `ProxyInstance` constructor changes the fetch size for all ``BatchScanner``s created +by that instance. If multiple fetch sizes are required/desired, the client application will have to create and manage multiple +``ProxyInstance``s and utilize the correct instance to create individual ``BatchScanner``s based on the fetch size requirements. + +The fetch size specified must be strictly greater than 0 and less than or equal to 2,000. If the value provided is outside of that +range, a warning will be logged and the default value of 1,000 will be used. + +To set a new fetch size via the constructor, use the 3-argument constructor: +[source,java] +---- + String host = "myhost"; + int port = 5432; + int fetchSize = 10; + + Instance inst = new ProxyInstance(host, port, fetchSize); + ... +---- + diff --git a/proxy-instance-docs/src/main/asciidoc/images/apl_horizontal_blue.png b/proxy-instance-docs/src/main/asciidoc/images/apl_horizontal_blue.png new file mode 100644 index 0000000..49787a1 Binary files /dev/null and b/proxy-instance-docs/src/main/asciidoc/images/apl_horizontal_blue.png differ diff --git a/proxy-instance-docs/src/main/asciidoc/images/proxy.png b/proxy-instance-docs/src/main/asciidoc/images/proxy.png new file mode 100644 index 0000000..c03c253 Binary files /dev/null and b/proxy-instance-docs/src/main/asciidoc/images/proxy.png differ diff --git a/proxy-instance-docs/src/main/asciidoc/images/traditional.png b/proxy-instance-docs/src/main/asciidoc/images/traditional.png new file mode 100644 index 0000000..3d5d621 Binary files /dev/null and b/proxy-instance-docs/src/main/asciidoc/images/traditional.png differ diff --git a/proxy-instance-docs/src/main/asciidoc/proxy_instance_user_manual.asciidoc b/proxy-instance-docs/src/main/asciidoc/proxy_instance_user_manual.asciidoc new file mode 100644 index 0000000..a559bf7 --- /dev/null +++ b/proxy-instance-docs/src/main/asciidoc/proxy_instance_user_manual.asciidoc @@ -0,0 +1,21 @@ += Accumulo Proxy Instance User Manual +:author: JHUAPL +:toc2: +:toclevels: 2 +:toc-title: Accumulo Proxy Instance +:numbered: +:revnumber: {docVersion} + +include::chapters/introduction.txt[] + +include::chapters/building.txt[] + +include::chapters/usage.txt[] + +include::chapters/development.txt[] + +include::chapters/known_issues.txt[] + +image::apl_horizontal_blue.png["JHU/APL",width="450",align="center"] + +include::chapters/copyright.txt[] diff --git a/proxy-instance/pom.xml b/proxy-instance/pom.xml new file mode 100644 index 0000000..741a0c8 --- /dev/null +++ b/proxy-instance/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + + edu.jhuapl.accumulo + proxy-instance-project + 1.0.0-SNAPSHOT + + proxy-instance + proxy-instance + Implements Accumulo Java API through the Accumulo Thrift-based Proxy server. + + 1.7 + 1.7 + + + + org.apache.accumulo + accumulo-core + ${accumulo.version} + + + org.apache.accumulo + accumulo-proxy + ${accumulo.version} + + + junit + junit + test + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + maven-assembly-plugin + + + + stec.va.core.Main + + + + jar-with-dependencies + + + + + com.googlecode.maven-java-formatter-plugin + maven-java-formatter-plugin + 0.4 + + ${project.basedir}/src/main/resources/Eclipse-Codestyle.xml + LF + + + + + format + + + + + + com.mycila + license-maven-plugin + 2.11 + +
license.txt
+ true + + **/pom.xml + **/resources/** + **/**.txt + **/**.asciidoc + LICENSE + + + + + + format + check + + process-resources + + + + + + diff --git a/proxy-instance/pom.xml.releaseBackup b/proxy-instance/pom.xml.releaseBackup new file mode 100644 index 0000000..244170e --- /dev/null +++ b/proxy-instance/pom.xml.releaseBackup @@ -0,0 +1,96 @@ + + + 4.0.0 + + edu.jhuapl.accumulo + proxy-instance-project + 1.0.0-SNAPSHOT + .. + + proxy-instance + proxy-instance + Implements Accumulo Java API through the Accumulo Thrift-based Proxy server. + + 1.7 + 1.7 + + + + org.apache.accumulo + accumulo-core + ${accumulo.version} + + + org.apache.accumulo + accumulo-proxy + ${accumulo.version} + + + junit + junit + test + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + maven-assembly-plugin + + + + stec.va.core.Main + + + + jar-with-dependencies + + + + + com.googlecode.maven-java-formatter-plugin + maven-java-formatter-plugin + 0.4 + + ${project.basedir}/src/main/resources/Eclipse-Codestyle.xml + LF + + + + + format + + + + + + com.mycila + license-maven-plugin + 2.11 + +
license.txt
+ true + + **/pom.xml + **/resources/** + **/**.txt + **/**.asciidoc + LICENSE + + + + + + format + check + + process-resources + + + + + + diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/AbstractProxyScanner.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/AbstractProxyScanner.java new file mode 100644 index 0000000..b67cc66 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/AbstractProxyScanner.java @@ -0,0 +1,90 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.ScannerBase; +import org.apache.accumulo.proxy.thrift.ScanColumn; +import org.apache.hadoop.io.Text; + +/** + * Parent class for proxy scanners. + */ +abstract class AbstractProxyScanner implements ScannerBase { + + /** + * The connector that created this scanner. + */ + protected ProxyConnector connector; + + /** + * The token used when making proxy requests. + */ + protected ByteBuffer token; + + /** + * Table name for this scanner. + */ + protected String tableName; + + /** + * Id assigned to this scanner by the proxy server. + */ + protected String scannerId = null; + + protected AbstractProxyScanner(ProxyConnector connector, ByteBuffer token, String tableName) { + this.connector = connector; + + this.token = token; + this.tableName = tableName; + } + + public void setTimeout(long timeOut, TimeUnit timeUnit) { + // proxy API does not support time outs for scanners + throw ExceptionFactory.unsupported(); + } + + public long getTimeout(TimeUnit timeUnit) { + // proxy API does not support time outs for scanners + throw ExceptionFactory.unsupported(); + } + + public void fetchColumnFamily(Text col) { + fetchColumn(col, null); + } + + public void fetchColumn(Text colFam, Text colQual) { + ScanColumn sc = new ScanColumn(); + if (colFam != null) { + sc.setColFamily(colFam.getBytes()); + } + if (colQual != null) { + sc.setColQualifier(colQual.getBytes()); + } + addToFetchOptions(sc); + } + + /** + * Subclasses must set themselves up to fetch the given ScanColumn. This allows Scanners and BatchScanners to handle the ScanColumn option differently. + * + * @param col + * the column to add to the current set of fetch options + */ + protected abstract void addToFetchOptions(ScanColumn col); + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ExceptionFactory.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ExceptionFactory.java new file mode 100644 index 0000000..09a26ff --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ExceptionFactory.java @@ -0,0 +1,113 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; + +/** + * A factory to create standard exceptions for specific conditions. + * */ +final class ExceptionFactory { + + /** + * As a utility, this should not be instantiated. + */ + private ExceptionFactory() {} + + /** + * Exception to throw when a Java Client API method is not supported in the Proxy Client API. + * + * @return a new UnsupportedOperationException that tells the user this feature is not supported by the Proxy Thrift interface. + */ + public static UnsupportedOperationException unsupported() { + return new UnsupportedOperationException("Operation not supported by Accumulo Proxy API."); + } + + /** + * Exception to throw when the ProxyInstace has not yet implemented a Java API method. + * + * @return a new UnsupportedOperationException that tells the user this feature is not implemented yet. + */ + + public static UnsupportedOperationException notYetImplemented() { + return new UnsupportedOperationException("Not yet implemented."); + } + + /** + * Creates an AccumuloException with the given Throwable embedded as the cause. + * + * @param cause + * the root cause of this exception + * @return a new AccumuloException with the given root cause + */ + public static AccumuloException accumuloException(Throwable cause) { + return new AccumuloException(cause); + } + + /** + * Wraps the thrift exception with a core client exception. + * + * @param tableName + * the table name that was not found. + * @param tnfe + * the Thrift TableNotFoundException + * @return a new (non-Thrift) TableNotFoundException + */ + public static TableNotFoundException tableNotFoundException(String tableName, org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + return new TableNotFoundException(null, tableName, tnfe.getMsg(), tnfe); + } + + /** + * Wraps the thrift exception with a core client exception. + * + * @param tableName + * the table name that already exists + * @param tee + * the Thrift TableExistsException + * @return a new (non-Thrift) TableExistsException + */ + public static TableExistsException tableExistsException(String tableName, org.apache.accumulo.proxy.thrift.TableExistsException tee) { + return new TableExistsException(null, tableName, tee.getMsg(), tee); + } + + /** + * Creates a new RuntimeException with the given Throwable as the cause. + * + * @param cause + * the cause of this exception + * @return a new RuntimeException with the given cause + */ + public static RuntimeException runtimeException(Throwable cause) { + return new RuntimeException(cause); + } + + /** + * Used when we expect to be able to convert a particular value from one enum type (e.g., Java API) in another enum type (e.g., Proxy API) but can't. As we + * expect this to always work, this is an Error, not an Exception. + * + * @param val + * the original enumeration value + * @param otherCls + * the other Enumeration type we were trying to which we were trying to map {@code val} + * @return a new ProxyInstanceError that informs the user we expected but were unable to find a valid enumeration mapping. + */ + public static ProxyInstanceError noEnumMapping(Enum val, Class> otherCls) { + return new ProxyInstanceError("Expected mapping from value " + val + " (" + val.getClass().getCanonicalName() + ") to " + otherCls.getCanonicalName() + + " but it does not exist."); + } +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/MutationBuffer.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/MutationBuffer.java new file mode 100644 index 0000000..eb6b077 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/MutationBuffer.java @@ -0,0 +1,177 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.proxy.thrift.ColumnUpdate; +import org.apache.accumulo.proxy.thrift.MutationsRejectedException; +import org.apache.thrift.TException; + +/** + * A class used to buffer mutation on the client side in a user-sized memory buffer and/or for a user-defined amount of time before sending to the + * ProxyInstance. By buffering and sending many requests at once, performance may be improved some situations. + * + * This class does not actually do the sending when thresholds are exceeded; rather it just provides the access to the buffered Mutations and tracks the time + * and memory in order to be able to inform other classes when the buffer need to be flushed. + * + */ +class MutationBuffer { + + /** + * The map of writerIds to a list of buffered mutations. + */ + private Map> mutations; + + /** + * The maximum amount of memory (in bytes) to use before automatically pushing the buffered mutations to the Proxy server. + */ + long maxMemory; + + /** + * The maximum amount of time (in milliseconds) to wait before automatically pushing the buffered mutations to the Proxy server. + */ + long maxLatencyMs; + + /** + * The current estimate of buffered mutation memory in bytes. + */ + long memory; + + ProxyConnector connector; + + Timer timer; + + /** + * Create a new MutationBuffer with the given configuration parameters. + * + * @param config + * the configuration for this buffer + */ + MutationBuffer(ProxyConnector connector, BatchWriterConfig config) { + this.connector = connector; + this.maxMemory = config.getMaxMemory(); + this.maxLatencyMs = config.getMaxLatency(TimeUnit.MILLISECONDS); + + this.memory = 0L; + this.mutations = new HashMap>(); + } + + /** + * Add a mutation to this buffer. + * + * @param mutation + * the mutation to add + * @throws org.apache.accumulo.core.client.MutationsRejectedException + * throws if the mutation cannot be accepted + */ + public synchronized void addMutation(String writerId, Mutation mutation) throws org.apache.accumulo.core.client.MutationsRejectedException { + if (mutations.isEmpty()) { + // this is the first entry... start watching latency... + timer = new Timer("", true); + timer.schedule(new TimerTask() { + + @Override + public void run() { + try { + latencyFlush(); + } catch (org.apache.accumulo.core.client.MutationsRejectedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + }, maxLatencyMs); + } + + // create our own copy... + mutation = new Mutation(mutation); + + List list = mutations.get(writerId); + if (list == null) { + list = new ArrayList(); + mutations.put(writerId, list); + } + list.add(mutation); + + memory += mutation.estimatedMemoryUsed(); + checkMemoryFlush(); + } + + void checkMemoryFlush() throws org.apache.accumulo.core.client.MutationsRejectedException { + if (memory >= maxMemory) { + // memory threshold exceeded + flush(); + } + } + + void latencyFlush() throws org.apache.accumulo.core.client.MutationsRejectedException { + if (mutations.size() != 0) { + // latency threshold exceeded + flush(); + } + } + + /** + * Called to flush the buffer. The method returns a map of rowIDs (as ByteBuffers) to lists of Thrift-based ColumnUpdates suitable for sending directly to the + * ProxyInstance. As a side effect, this method also resets this buffer. + * + * @throws org.apache.accumulo.core.client.MutationsRejectedException + * thrown if any buffered mutations are rejected while flushing + */ + public synchronized void flush() throws org.apache.accumulo.core.client.MutationsRejectedException { + for (Entry> entry : mutations.entrySet()) { + String writerId = entry.getKey(); + Map> updates = new HashMap>(); + for (Mutation m : entry.getValue()) { + ByteBuffer key = ByteBuffer.wrap(m.getRow()); + List updateList = updates.get(key); + if (updateList == null) { + updateList = new ArrayList(); + updates.put(key, updateList); + } + ThriftHelper.addThriftColumnUpdates(updateList, m.getUpdates()); + } + try { + connector.getClient().update(writerId, updates); + connector.getClient().flush(writerId); + } catch (MutationsRejectedException mre) { + throw ThriftHelper.fromThrift(mre, connector.getInstance()); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + // and reset... + memory = 0; + if (timer != null) { + timer.cancel(); + timer = null; + } + mutations.clear(); + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveCompaction.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveCompaction.java new file mode 100644 index 0000000..06a7153 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveCompaction.java @@ -0,0 +1,121 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.util.List; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.ActiveCompaction; +import org.apache.accumulo.core.data.KeyExtent; + +/** + * Implementation of an ActiveCompaction for the ProxyInstance. This is basically a data structure to hold all of the provided details about an active + * compaction. + */ +class ProxyActiveCompaction extends ActiveCompaction { + + String table; + KeyExtent extent; + long age; + List inputFiles; + String outputFile; + CompactionType type; + CompactionReason reason; + String localityGroup; + long entriesRead; + long entriesWritten; + List iterators; + + ProxyActiveCompaction(String table, KeyExtent extent, long age, List inputFiles, String outputFile, CompactionType type, CompactionReason reason, + String localityGroup, long entriesRead, long entriesWritten, List iterators) { + super(); + this.table = table; + this.extent = extent; + this.age = age; + this.inputFiles = inputFiles; + this.outputFile = outputFile; + this.type = type; + this.reason = reason; + this.localityGroup = localityGroup; + this.entriesRead = entriesRead; + this.entriesWritten = entriesWritten; + this.iterators = iterators; + } + + @Override + public String getTable() throws TableNotFoundException { + return table; + } + + @Override + public KeyExtent getExtent() { + return extent; + } + + @Override + public long getAge() { + return age; + } + + @Override + public List getInputFiles() { + return inputFiles; + } + + @Override + public String getOutputFile() { + return outputFile; + } + + @Override + public CompactionType getType() { + return type; + } + + @Override + public CompactionReason getReason() { + return reason; + } + + @Override + public String getLocalityGroup() { + return localityGroup; + } + + @Override + public long getEntriesRead() { + return entriesRead; + } + + @Override + public long getEntriesWritten() { + return entriesWritten; + } + + @Override + public List getIterators() { + return iterators; + } + + @Override + public String toString() { + return "ProxyActiveCompaction [table=" + table + ", extent=" + extent + ", age=" + age + ", inputFiles=" + inputFiles + ", outputFile=" + outputFile + + ", type=" + type + ", reason=" + reason + ", localityGroup=" + localityGroup + ", entriesRead=" + entriesRead + ", entriesWritten=" + entriesWritten + + ", iterators=" + iterators + "]"; + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveScan.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveScan.java new file mode 100644 index 0000000..7d3fcb9 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyActiveScan.java @@ -0,0 +1,141 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.admin.ActiveScan; +import org.apache.accumulo.core.client.admin.ScanState; +import org.apache.accumulo.core.client.admin.ScanType; +import org.apache.accumulo.core.data.Column; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.security.Authorizations; + +/** + * Implementation of an ActiveScan for the ProxyInstance. This is basically a data structure to hold all of the provided details about an active scan. + */ +class ProxyActiveScan extends ActiveScan { + + long scanId; + String client; + String user; + String table; + long age; + long lastContactTime; + ScanType type; + ScanState state; + KeyExtent extent; + List columns; + Authorizations authorizations; + long idleTime; + + ProxyActiveScan(long scanId, String client, String user, String table, long age, long lastContactTime, ScanType type, ScanState state, KeyExtent extent, + List columns, List authorizations, long idleTime) { + super(); + this.scanId = scanId; + this.client = client; + this.user = user; + this.table = table; + this.age = age; + this.lastContactTime = lastContactTime; + this.type = type; + this.state = state; + this.extent = extent; + this.columns = columns; + this.authorizations = new Authorizations(authorizations); + this.idleTime = idleTime; + } + + @Override + public long getScanid() { + return scanId; + } + + @Override + public String getClient() { + return client; + } + + @Override + public String getUser() { + return user; + } + + @Override + public String getTable() { + return table; + } + + @Override + public long getAge() { + return age; + } + + @Override + public long getLastContactTime() { + return lastContactTime; + } + + @Override + public ScanType getType() { + return type; + } + + @Override + public ScanState getState() { + return state; + } + + @Override + public KeyExtent getExtent() { + return extent; + } + + @Override + public List getColumns() { + return columns; + } + + @Override + public List getSsiList() { + throw ExceptionFactory.unsupported(); + } + + @Override + public Map> getSsio() { + throw ExceptionFactory.unsupported(); + } + + @Override + public Authorizations getAuthorizations() { + return authorizations; + } + + @Override + public long getIdleTime() { + return idleTime; + } + + @Override + public String toString() { + return "ProxyActiveScan [scanId=" + scanId + ", client=" + client + ", user=" + user + ", table=" + table + ", age=" + age + ", lastContactTime=" + + lastContactTime + ", type=" + type + ", state=" + state + ", extent=" + extent + ", columns=" + columns + ", authorizations=" + authorizations + + ", idleTime=" + idleTime + "]"; + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchScanner.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchScanner.java new file mode 100644 index 0000000..c8bf048 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchScanner.java @@ -0,0 +1,134 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.accumulo.proxy.thrift.BatchScanOptions; +import org.apache.accumulo.proxy.thrift.ScanColumn; +import org.apache.thrift.TException; +import org.slf4j.LoggerFactory; + +/** + * Implementation of a BatchScanner for the ProxyInstance. + * + */ +class ProxyBatchScanner extends AbstractProxyScanner implements BatchScanner { + + /** + * The scan options set for this batch scanner. + */ + protected BatchScanOptions batchOptions; + + protected int fetchSize; + + ProxyBatchScanner(ProxyConnector connector, ByteBuffer token, String tableName, Authorizations authorizations, int numQueryThreads, int fetchSize) { + super(connector, token, tableName); + this.fetchSize = fetchSize; + + batchOptions = new BatchScanOptions(); + batchOptions.setThreads(numQueryThreads); + for (ByteBuffer auth : authorizations.getAuthorizationsBB()) { + batchOptions.addToAuthorizations(auth); + } + } + + public void setRanges(Collection ranges) { + if (ranges == null) { + batchOptions.unsetRanges(); + } else { + batchOptions.setRanges(ThriftHelper.toThriftRanges(ranges)); + } + } + + public void addScanIterator(IteratorSetting cfg) { + batchOptions.addToIterators(ThriftHelper.toThrift(cfg)); + } + + public void removeScanIterator(String iteratorName) { + for (org.apache.accumulo.proxy.thrift.IteratorSetting is : batchOptions.iterators) { + if (is.getName().equals(iteratorName)) { + batchOptions.iterators.remove(is); + break; + } + } + } + + public void updateScanIteratorOption(String iteratorName, String key, String value) { + for (org.apache.accumulo.proxy.thrift.IteratorSetting is : batchOptions.iterators) { + if (is.getName().equals(iteratorName)) { + is.putToProperties(key, value); + } + } + } + + @Override + protected void addToFetchOptions(ScanColumn col) { + batchOptions.addToColumns(col); + } + + public void clearColumns() { + if (batchOptions.getColumns() != null) { + batchOptions.getColumns().clear(); + } + } + + public void clearScanIterators() { + if (batchOptions.getIterators() != null) { + batchOptions.getIterators().clear(); + } + } + + public Iterator> iterator() { + AccumuloProxy.Iface client = connector.getClient(); + try { + scannerId = client.createBatchScanner(token, tableName, batchOptions); + return new ScannerIterator(scannerId, connector, fetchSize); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public void close() { + try { + connector.getClient().closeScanner(scannerId); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } finally { + scannerId = null; + } + } + + @Override + protected void finalize() { + if (scannerId != null) { + close(); + LoggerFactory.getLogger(ProxyBatchScanner.class).warn( + "BatchScanner " + scannerId + " in finalize but not closed; " + "you forgot to close a batch scanner!"); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchWriter.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchWriter.java new file mode 100644 index 0000000..6edc06b --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyBatchWriter.java @@ -0,0 +1,148 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.proxy.thrift.AccumuloException; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.accumulo.proxy.thrift.AccumuloSecurityException; +import org.apache.accumulo.proxy.thrift.TableNotFoundException; +import org.apache.accumulo.proxy.thrift.UnknownWriter; +import org.apache.accumulo.proxy.thrift.WriterOptions; +import org.apache.thrift.TException; +import org.slf4j.LoggerFactory; + +/** + * Implementation of a BatchWriter to a Proxy Server. + */ +class ProxyBatchWriter implements BatchWriter { + + /** + * Parent connector. + */ + ProxyConnector connector; + + /** + * Local copy of connector's Thrift RPC object to Proxy Server. + */ + private AccumuloProxy.Iface client; + + /** + * Token used when communicating with the proxy server. + */ + ByteBuffer token; + + /** + * The id for this writer. + */ + String writerId; + + /** + * An internal buffer to provide in-memory, time-based buffering of mutations to send in batches. This uses the same memory and latency parameters the actual + * BatchWriter will use on the Proxy Server side as well. + */ + private MutationBuffer mutationBuffer; + + private boolean closed; + + ProxyBatchWriter(ProxyConnector connector, ByteBuffer token, String table, BatchWriterConfig config) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException, TException { + this.connector = connector; + this.client = connector.getClient(); + + this.token = token; + WriterOptions opts = new WriterOptions(); + opts.setLatencyMs(config.getMaxLatency(TimeUnit.MILLISECONDS)); + opts.setMaxMemory(config.getMaxMemory()); + opts.setThreads(config.getMaxWriteThreads()); + writerId = client.createWriter(token, table, opts); + + mutationBuffer = new MutationBuffer(connector, config); + closed = false; + } + + private void checkClosed() throws IllegalStateException { + if (closed) { + throw new IllegalStateException("Closed."); + } + } + + @Override + public void addMutation(Mutation m) throws MutationsRejectedException { + checkClosed(); + addMutationNoCloseCheck(m); + } + + @Override + public void addMutations(Iterable iterable) throws MutationsRejectedException { + checkClosed(); + for (Mutation m : iterable) { + addMutationNoCloseCheck(m); + } + } + + private void addMutationNoCloseCheck(Mutation m) throws MutationsRejectedException { + if (m.size() == 0) { + throw new IllegalArgumentException("Cannot add empty mutation."); + } + + mutationBuffer.addMutation(writerId, m); + } + + @Override + public void flush() throws MutationsRejectedException { + checkClosed(); + mutationBuffer.flush(); + } + + @Override + public void close() throws MutationsRejectedException { + checkClosed(); + mutationBuffer.flush(); + + try { + client.closeWriter(writerId); + closed = true; + } catch (UnknownWriter e) { + throw ExceptionFactory.runtimeException(e); + } catch (org.apache.accumulo.proxy.thrift.MutationsRejectedException e) { + throw ThriftHelper.fromThrift(e, connector.getInstance()); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + @Override + protected void finalize() { + if (!closed) { + LoggerFactory.getLogger(ProxyBatchWriter.class).warn( + "ProxyBatchWriter ID " + writerId + " in finalize but not closed; " + "you forgot to close a ProxyBatchWriter!"); + + try { + close(); + } catch (MutationsRejectedException mre) { + LoggerFactory.getLogger(ProxyBatchScanner.class).warn("Problem closing ProxyMultiTableBatchWriter.", mre); + } + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConditionalWriter.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConditionalWriter.java new file mode 100644 index 0000000..dd5d5fb --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConditionalWriter.java @@ -0,0 +1,126 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.ConditionalWriter; +import org.apache.accumulo.core.client.ConditionalWriterConfig; +import org.apache.accumulo.core.data.ConditionalMutation; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.accumulo.proxy.thrift.ColumnUpdate; +import org.apache.accumulo.proxy.thrift.ConditionalStatus; +import org.apache.accumulo.proxy.thrift.ConditionalUpdates; +import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions; +import org.apache.thrift.TException; + +class ProxyConditionalWriter implements ConditionalWriter { + + AccumuloProxy.Iface client; + + String writerId; + + ProxyConditionalWriter(ProxyConnector connector, ByteBuffer token, String table, ConditionalWriterConfig config) { + this.client = connector.getClient(); + + ConditionalWriterOptions options = new ConditionalWriterOptions(); + options.setAuthorizations(new HashSet(config.getAuthorizations().getAuthorizationsBB())); + options.setThreads(config.getMaxWriteThreads()); + options.setTimeoutMs(config.getTimeout(TimeUnit.MILLISECONDS)); + try { + writerId = client.createConditionalWriter(token, table, options); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + + } + + public Iterator write(Iterator mutations) { + Map updates = new HashMap(); + Map mutationMap = new HashMap(); + + while (mutations.hasNext()) { + ConditionalMutation mutation = mutations.next(); + ByteBuffer key = ByteBuffer.wrap(mutation.getRow()); + ConditionalUpdates update = updates.get(key); + + // NOTE: API seems to imply you will provide a mutation against + // any single row ID at most once within this iterator or + // conditional mutations. Otherwise, the mutations and the + // conditions all get co-mingled when placed in the parameter + // map! TODO- should we check for this and throw an exception? + + // working under the assumption you do not want to co-mingle updates + // for a single row within a single call here, we will *replace* + // (vs. update) anything that was existing in the map. This feels + // wrong... + + update = new ConditionalUpdates(); + updates.put(key, update); + mutationMap.put(key, mutation); + update.setConditions(ThriftHelper.toThriftCondition(mutation.getConditions())); + List tupdates = new LinkedList(); + ThriftHelper.addThriftColumnUpdates(tupdates, mutation.getUpdates()); + update.setUpdates(tupdates); + } + + try { + Map status = client.updateRowsConditionally(writerId, updates); + + List results = new LinkedList(); + for (Entry entry : status.entrySet()) { + ConditionalMutation cm = mutationMap.get(entry.getKey()); + Status staus = ThriftHelper.convertEnum(entry.getValue(), Status.class); + Result r = new Result(staus, cm, null) { + + @Override + public String getTabletServer() { + // we are not given the tablet server nor have any way + // to find out what it is... + throw ExceptionFactory.unsupported(); + } + }; + results.add(r); + } + return results.iterator(); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public Result write(ConditionalMutation mutation) { + Iterator results = write(Collections.singleton(mutation).iterator()); + return results.next(); + } + + public void close() { + try { + client.closeConditionalWriter(writerId); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConnector.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConnector.java new file mode 100644 index 0000000..f534cb3 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyConnector.java @@ -0,0 +1,170 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import static edu.jhuapl.accumulo.proxy.ThriftHelper.UTF8; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.ConditionalWriter; +import org.apache.accumulo.core.client.ConditionalWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.MultiTableBatchWriter; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.InstanceOperations; +import org.apache.accumulo.core.client.admin.NamespaceOperations; +import org.apache.accumulo.core.client.admin.SecurityOperations; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.accumulo.proxy.thrift.AccumuloSecurityException; +import org.apache.thrift.TException; + +class ProxyConnector extends Connector { + + ProxyInstance instance; + String principal; + ByteBuffer token; + + ProxyConnector(ProxyInstance instance, String principal, AuthenticationToken auth) throws AccumuloSecurityException, TException { + // TODO probably a better way to do this... + if (!(auth instanceof PasswordToken)) { + throw new IllegalArgumentException("Currently only works with PasswordTokens."); + } + + this.instance = instance; + this.principal = principal; + String passwd = new String(((PasswordToken) auth).getPassword(), UTF8); + Map password = new HashMap(); + password.put("password", passwd); + token = instance.getClient().login(principal, password); + } + + @Override + public BatchScanner createBatchScanner(String tableName, Authorizations authorizations, int numQueryThreads) throws TableNotFoundException { + if (!tableOperations().exists(tableName)) { + throw new TableNotFoundException(null, tableName, null); + } + return new ProxyBatchScanner(this, token, tableName, authorizations, numQueryThreads, instance.getBatchScannerFetchSize()); + } + + @Override + @Deprecated + public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, long maxMemory, long maxLatency, + int maxWriteThreads) throws TableNotFoundException { + return createBatchDeleter(tableName, authorizations, numQueryThreads, new BatchWriterConfig().setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) + .setMaxMemory(maxMemory).setMaxWriteThreads(maxWriteThreads)); + } + + @Override + public BatchDeleter createBatchDeleter(String tableName, Authorizations authorizations, int numQueryThreads, BatchWriterConfig config) + throws TableNotFoundException { + throw ExceptionFactory.notYetImplemented(); + } + + @Override + @Deprecated + public BatchWriter createBatchWriter(String tableName, long maxMemory, long maxLatency, int maxWriteThreads) throws TableNotFoundException { + return createBatchWriter(tableName, + new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(maxWriteThreads)); + } + + @Override + public BatchWriter createBatchWriter(String tableName, BatchWriterConfig config) throws TableNotFoundException { + if (!tableOperations().exists(tableName)) { + throw new TableNotFoundException(null, tableName, null); + } + + try { + return new ProxyBatchWriter(this, token, tableName, config); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + @Override + @Deprecated + public MultiTableBatchWriter createMultiTableBatchWriter(long maxMemory, long maxLatency, int maxWriteThreads) { + return createMultiTableBatchWriter(new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS) + .setMaxWriteThreads(maxWriteThreads)); + } + + @Override + public MultiTableBatchWriter createMultiTableBatchWriter(BatchWriterConfig config) { + return new ProxyMultiTableBatchWriter(this, this.token, config); + } + + @Override + public Scanner createScanner(String tableName, Authorizations authorizations) throws TableNotFoundException { + if (!tableOperations().exists(tableName)) { + throw new TableNotFoundException(null, tableName, null); + } + return new ProxyScanner(this, token, tableName, authorizations); + } + + @Override + public ConditionalWriter createConditionalWriter(String tableName, ConditionalWriterConfig config) throws TableNotFoundException { + if (!tableOperations().exists(tableName)) { + throw new TableNotFoundException(null, tableName, null); + } + return new ProxyConditionalWriter(this, token, tableName, config); + } + + AccumuloProxy.Iface getClient() { + return instance.getClient(); + } + + @Override + public Instance getInstance() { + return instance; + } + + @Override + public String whoami() { + return principal; + } + + @Override + public TableOperations tableOperations() { + return new ProxyTableOperations(this, token); + } + + @Override + public NamespaceOperations namespaceOperations() { + throw ExceptionFactory.unsupported(); + } + + @Override + public SecurityOperations securityOperations() { + return new ProxySecurityOperations(this, token); + } + + @Override + public InstanceOperations instanceOperations() { + return new ProxyInstanceOperations(this, token); + } +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstance.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstance.java new file mode 100644 index 0000000..515b0e3 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstance.java @@ -0,0 +1,722 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.io.Closeable; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.accumulo.proxy.thrift.ActiveCompaction; +import org.apache.accumulo.proxy.thrift.ActiveScan; +import org.apache.accumulo.proxy.thrift.BatchScanOptions; +import org.apache.accumulo.proxy.thrift.ColumnUpdate; +import org.apache.accumulo.proxy.thrift.ConditionalStatus; +import org.apache.accumulo.proxy.thrift.ConditionalUpdates; +import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions; +import org.apache.accumulo.proxy.thrift.DiskUsage; +import org.apache.accumulo.proxy.thrift.IteratorScope; +import org.apache.accumulo.proxy.thrift.IteratorSetting; +import org.apache.accumulo.proxy.thrift.Key; +import org.apache.accumulo.proxy.thrift.KeyValueAndPeek; +import org.apache.accumulo.proxy.thrift.MutationsRejectedException; +import org.apache.accumulo.proxy.thrift.NoMoreEntriesException; +import org.apache.accumulo.proxy.thrift.PartialKey; +import org.apache.accumulo.proxy.thrift.Range; +import org.apache.accumulo.proxy.thrift.ScanOptions; +import org.apache.accumulo.proxy.thrift.ScanResult; +import org.apache.accumulo.proxy.thrift.SystemPermission; +import org.apache.accumulo.proxy.thrift.TableExistsException; +import org.apache.accumulo.proxy.thrift.TableNotFoundException; +import org.apache.accumulo.proxy.thrift.TablePermission; +import org.apache.accumulo.proxy.thrift.TimeType; +import org.apache.accumulo.proxy.thrift.UnknownScanner; +import org.apache.accumulo.proxy.thrift.UnknownWriter; +import org.apache.accumulo.proxy.thrift.WriterOptions; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TFramedTransport; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.LoggerFactory; + +/** + * A proxy instance that uses the Accumulo Proxy Thrift interface to fulfill the Accumulo client APIs. Note this instance implements Closeable and, while not + * part of the public Instance API, should be closed when no longer needed. + */ +public class ProxyInstance implements Instance, Closeable { + + /** + * Default fetch size to be used for BatchScanners. Currently equal to 1,000. + */ + private static final int DEFAULT_FETCH_SIZE = 1_000; + + private AccumuloProxy.Iface client; + + private TTransport transport; + private int fetchSize; + + /** + * Assumes a TSocket transport wrapped by a TFramedTransport. + * + * @param host + * the host name or IP address where the Accumulo Thrift Proxy server is running + * @param port + * the port where the Accumulo Thrift Proxy server is listening + * @throws TTransportException + * thrown if the Thrift TTransport cannot be established. + */ + public ProxyInstance(String host, int port) throws TTransportException { + this(host, port, DEFAULT_FETCH_SIZE); + } + + /** + * Assumes a TSocket transport wrapped by a TFramedTransport. + * + * @param host + * the host name or IP address where the Accumulo Thrift Proxy server is running + * @param port + * the port where the Accumulo Thrift Proxy server is listening + * @param fetchSize + * the fetch size for BatchScanners + * @throws TTransportException + * thrown if the Thrift TTransport cannot be established. + */ + public ProxyInstance(String host, int port, int fetchSize) throws TTransportException { + this(new TFramedTransport(new TSocket(host, port)), fetchSize); + } + + public ProxyInstance(TTransport transport) throws TTransportException { + this(transport, DEFAULT_FETCH_SIZE); + } + + /** + * + * @param transport + * Thrift transport to communicate with Proxy Server + * @param fetchSize + * the fetch size for BatchScanners. Must be 0 < fetchSize <= 2000. If fetchSize is outside of this range, a warning will be logged and the + * {@link #DEFAULT_FETCH_SIZE} will be used. + * @throws TTransportException + * thrown if the Thrift TTransport cannot be established. + */ + public ProxyInstance(TTransport transport, int fetchSize) throws TTransportException { + if (!transport.isOpen()) { + transport.open(); + } + TProtocol protocol = new TCompactProtocol(transport); + client = new SynchronizedProxy(new AccumuloProxy.Client(protocol)); + this.transport = transport; + + if (fetchSize <= 0 || fetchSize > 2000) { + LoggerFactory.getLogger(ProxyInstance.class).warn( + "Fetch size out of range (0 < fetchSize <= 2000): " + fetchSize + "; using default: " + DEFAULT_FETCH_SIZE); + this.fetchSize = DEFAULT_FETCH_SIZE; + } else { + this.fetchSize = fetchSize; + } + } + + public int getBatchScannerFetchSize() { + return fetchSize; + } + + public String getRootTabletLocation() { + throw ExceptionFactory.unsupported(); + } + + public List getMasterLocations() { + throw ExceptionFactory.unsupported(); + } + + public String getInstanceID() { + throw ExceptionFactory.unsupported(); + } + + public String getInstanceName() { + throw ExceptionFactory.unsupported(); + } + + public String getZooKeepers() { + throw ExceptionFactory.unsupported(); + } + + public int getZooKeepersSessionTimeOut() { + throw ExceptionFactory.unsupported(); + } + + @Deprecated + public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, new PasswordToken(pass)); + } + + @Deprecated + public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, new PasswordToken(pass)); + } + + @Deprecated + public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException { + return getConnector(user, new PasswordToken(pass)); + } + + @Deprecated + public AccumuloConfiguration getConfiguration() { + throw ExceptionFactory.unsupported(); + } + + @Deprecated + public void setConfiguration(AccumuloConfiguration conf) { + throw ExceptionFactory.unsupported(); + } + + public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { + try { + return new ProxyConnector(this, principal, token); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + AccumuloProxy.Iface getClient() { + return client; + } + + public void close() { + // TODO- Neither Instance API nor Connector have a "close" method. But + // we need to close the transport when done. How to handle it? Currently + // clients must cast their Instance as a ProxyInstance and call close, + // *hope* the finalize method is called, or just be a bad citizen and + // not clean up resources on the proxy server. + if (transport != null) { + try { + transport.close(); + } finally { + transport = null; + client = null; + } + } + } + + @Override + protected void finalize() { + close(); + } + + /** + * A wrapper class that synchronizes every method in the Iface interface against this object, and then passes the call through to the wrapped Iface delegate. + * Due to the asynchronous nature of Thrift internals, if multiple threads use the same Iface object, the server may receive requests out-of-order and fail. + * This class helps mitigate that risk by ensuring only one thread is ever communicating with the proxy at one time. This incurs a performance hit, but if you + * are using the proxy instance, you probably aren't too concerned about throughput anyway... + */ + private static class SynchronizedProxy implements AccumuloProxy.Iface { + + AccumuloProxy.Iface delegate; + + SynchronizedProxy(AccumuloProxy.Iface delegate) { + this.delegate = delegate; + } + + @Override + public synchronized ByteBuffer login(String principal, Map loginProperties) + throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.login(principal, loginProperties); + } + + @Override + public synchronized int addConstraint(ByteBuffer login, String tableName, String constraintClassName) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.addConstraint(login, tableName, constraintClassName); + } + + @Override + public synchronized void addSplits(ByteBuffer login, String tableName, Set splits) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.addSplits(login, tableName, splits); + } + + @Override + public synchronized void attachIterator(ByteBuffer login, String tableName, IteratorSetting setting, Set scopes) + throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.AccumuloException, TableNotFoundException, + TException { + delegate.attachIterator(login, tableName, setting, scopes); + } + + @Override + public synchronized void checkIteratorConflicts(ByteBuffer login, String tableName, IteratorSetting setting, Set scopes) + throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, org.apache.accumulo.proxy.thrift.AccumuloException, TableNotFoundException, + TException { + delegate.checkIteratorConflicts(login, tableName, setting, scopes); + } + + @Override + public synchronized void clearLocatorCache(ByteBuffer login, String tableName) throws TableNotFoundException, TException { + delegate.clearLocatorCache(login, tableName); + } + + @Override + public synchronized void cloneTable(ByteBuffer login, String tableName, String newTableName, boolean flush, Map propertiesToSet, + Set propertiesToExclude) throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, + TableNotFoundException, TableExistsException, TException { + delegate.cloneTable(login, tableName, newTableName, flush, propertiesToSet, propertiesToExclude); + } + + @Override + public synchronized void compactTable(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow, List iterators, + boolean flush, boolean wait) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + org.apache.accumulo.proxy.thrift.AccumuloException, TException { + delegate.compactTable(login, tableName, startRow, endRow, iterators, flush, wait); + } + + @Override + public synchronized void cancelCompaction(ByteBuffer login, String tableName) throws org.apache.accumulo.proxy.thrift.AccumuloSecurityException, + TableNotFoundException, org.apache.accumulo.proxy.thrift.AccumuloException, TException { + delegate.cancelCompaction(login, tableName); + } + + @Override + public synchronized void createTable(ByteBuffer login, String tableName, boolean versioningIter, TimeType type) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableExistsException, TException { + delegate.createTable(login, tableName, versioningIter, type); + } + + @Override + public synchronized void deleteTable(ByteBuffer login, String tableName) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.deleteTable(login, tableName); + } + + @Override + public synchronized void deleteRows(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.deleteRows(login, tableName, startRow, endRow); + } + + @Override + public synchronized void exportTable(ByteBuffer login, String tableName, String exportDir) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.exportTable(login, tableName, exportDir); + } + + @Override + public synchronized void flushTable(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow, boolean wait) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.flushTable(login, tableName, startRow, endRow, wait); + } + + @Override + public synchronized List getDiskUsage(ByteBuffer login, Set tables) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.getDiskUsage(login, tables); + } + + @Override + public synchronized Map> getLocalityGroups(ByteBuffer login, String tableName) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.getLocalityGroups(login, tableName); + } + + @Override + public synchronized IteratorSetting getIteratorSetting(ByteBuffer login, String tableName, String iteratorName, IteratorScope scope) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.getIteratorSetting(login, tableName, iteratorName, scope); + } + + @Override + public synchronized ByteBuffer getMaxRow(ByteBuffer login, String tableName, Set auths, ByteBuffer startRow, boolean startInclusive, + ByteBuffer endRow, boolean endInclusive) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.getMaxRow(login, tableName, auths, startRow, startInclusive, endRow, endInclusive); + } + + @Override + public synchronized Map getTableProperties(ByteBuffer login, String tableName) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.getTableProperties(login, tableName); + } + + @Override + public synchronized void importDirectory(ByteBuffer login, String tableName, String importDir, String failureDir, boolean setTime) + throws TableNotFoundException, org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, + TException { + delegate.importDirectory(login, tableName, importDir, failureDir, setTime); + } + + @Override + public synchronized void importTable(ByteBuffer login, String tableName, String importDir) throws TableExistsException, + org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.importTable(login, tableName, importDir); + } + + @Override + public synchronized List listSplits(ByteBuffer login, String tableName, int maxSplits) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.listSplits(login, tableName, maxSplits); + } + + @Override + public synchronized Set listTables(ByteBuffer login) throws TException { + return delegate.listTables(login); + } + + @Override + public synchronized Map> listIterators(ByteBuffer login, String tableName) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.listIterators(login, tableName); + } + + @Override + public synchronized Map listConstraints(ByteBuffer login, String tableName) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.listConstraints(login, tableName); + } + + @Override + public synchronized void mergeTablets(ByteBuffer login, String tableName, ByteBuffer startRow, ByteBuffer endRow) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.mergeTablets(login, tableName, startRow, endRow); + } + + @Override + public synchronized void offlineTable(ByteBuffer login, String tableName, boolean wait) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.offlineTable(login, tableName, wait); + } + + @Override + public synchronized void onlineTable(ByteBuffer login, String tableName, boolean wait) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.onlineTable(login, tableName, wait); + } + + @Override + public synchronized void removeConstraint(ByteBuffer login, String tableName, int constraint) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + delegate.removeConstraint(login, tableName, constraint); + } + + @Override + public synchronized void removeIterator(ByteBuffer login, String tableName, String iterName, Set scopes) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.removeIterator(login, tableName, iterName, scopes); + } + + @Override + public synchronized void removeTableProperty(ByteBuffer login, String tableName, String property) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.removeTableProperty(login, tableName, property); + } + + @Override + public synchronized void renameTable(ByteBuffer login, String oldTableName, String newTableName) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TableExistsException, TException { + delegate.renameTable(login, oldTableName, newTableName); + } + + @Override + public synchronized void setLocalityGroups(ByteBuffer login, String tableName, Map> groups) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.setLocalityGroups(login, tableName, groups); + } + + @Override + public synchronized void setTableProperty(ByteBuffer login, String tableName, String property, String value) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.setTableProperty(login, tableName, property, value); + } + + @Override + public synchronized Set splitRangeByTablets(ByteBuffer login, String tableName, Range range, int maxSplits) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.splitRangeByTablets(login, tableName, range, maxSplits); + } + + @Override + public synchronized boolean tableExists(ByteBuffer login, String tableName) throws TException { + return delegate.tableExists(login, tableName); + } + + @Override + public synchronized Map tableIdMap(ByteBuffer login) throws TException { + return delegate.tableIdMap(login); + } + + @Override + public synchronized boolean testTableClassLoad(ByteBuffer login, String tableName, String className, String asTypeName) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.testClassLoad(login, className, asTypeName); + } + + @Override + public synchronized void pingTabletServer(ByteBuffer login, String tserver) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.pingTabletServer(login, tserver); + } + + @Override + public synchronized List getActiveScans(ByteBuffer login, String tserver) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.getActiveScans(login, tserver); + } + + @Override + public synchronized List getActiveCompactions(ByteBuffer login, String tserver) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.getActiveCompactions(login, tserver); + } + + @Override + public synchronized Map getSiteConfiguration(ByteBuffer login) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.getSiteConfiguration(login); + } + + @Override + public synchronized Map getSystemConfiguration(ByteBuffer login) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.getSystemConfiguration(login); + } + + @Override + public synchronized List getTabletServers(ByteBuffer login) throws TException { + return delegate.getTabletServers(login); + } + + @Override + public synchronized void removeProperty(ByteBuffer login, String property) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.removeProperty(login, property); + } + + @Override + public synchronized void setProperty(ByteBuffer login, String property, String value) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.setProperty(login, property, value); + } + + @Override + public synchronized boolean testClassLoad(ByteBuffer login, String className, String asTypeName) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.testClassLoad(login, className, asTypeName); + } + + @Override + public synchronized boolean authenticateUser(ByteBuffer login, String user, Map properties) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.authenticateUser(login, user, properties); + } + + @Override + public synchronized void changeUserAuthorizations(ByteBuffer login, String user, Set authorizations) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.changeUserAuthorizations(login, user, authorizations); + } + + @Override + public synchronized void changeLocalUserPassword(ByteBuffer login, String user, ByteBuffer password) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.changeLocalUserPassword(login, user, password); + } + + @Override + public synchronized void createLocalUser(ByteBuffer login, String user, ByteBuffer password) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.createLocalUser(login, user, password); + } + + @Override + public synchronized void dropLocalUser(ByteBuffer login, String user) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.dropLocalUser(login, user); + } + + @Override + public synchronized List getUserAuthorizations(ByteBuffer login, String user) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.getUserAuthorizations(login, user); + } + + @Override + public synchronized void grantSystemPermission(ByteBuffer login, String user, SystemPermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.grantSystemPermission(login, user, perm); + } + + @Override + public synchronized void grantTablePermission(ByteBuffer login, String user, String table, TablePermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.grantTablePermission(login, user, table, perm); + } + + @Override + public synchronized boolean hasSystemPermission(ByteBuffer login, String user, SystemPermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.hasSystemPermission(login, user, perm); + } + + @Override + public synchronized boolean hasTablePermission(ByteBuffer login, String user, String table, TablePermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.hasTablePermission(login, user, table, perm); + } + + @Override + public synchronized Set listLocalUsers(ByteBuffer login) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.listLocalUsers(login); + } + + @Override + public synchronized void revokeSystemPermission(ByteBuffer login, String user, SystemPermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + delegate.revokeSystemPermission(login, user, perm); + } + + @Override + public synchronized void revokeTablePermission(ByteBuffer login, String user, String table, TablePermission perm) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + delegate.revokeTablePermission(login, user, table, perm); + } + + @Override + public synchronized String createBatchScanner(ByteBuffer login, String tableName, BatchScanOptions options) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.createBatchScanner(login, tableName, options); + } + + @Override + public synchronized String createScanner(ByteBuffer login, String tableName, ScanOptions options) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.createScanner(login, tableName, options); + } + + @Override + public synchronized boolean hasNext(String scanner) throws UnknownScanner, TException { + return delegate.hasNext(scanner); + } + + @Override + public synchronized KeyValueAndPeek nextEntry(String scanner) throws NoMoreEntriesException, UnknownScanner, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.nextEntry(scanner); + } + + @Override + public synchronized ScanResult nextK(String scanner, int k) throws NoMoreEntriesException, UnknownScanner, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.nextK(scanner, k); + } + + @Override + public synchronized void closeScanner(String scanner) throws UnknownScanner, TException { + delegate.closeScanner(scanner); + } + + @Override + public synchronized void updateAndFlush(ByteBuffer login, String tableName, Map> cells) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + MutationsRejectedException, TException { + delegate.updateAndFlush(login, tableName, cells); + } + + @Override + public synchronized String createWriter(ByteBuffer login, String tableName, WriterOptions opts) throws org.apache.accumulo.proxy.thrift.AccumuloException, + org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, TException { + return delegate.createWriter(login, tableName, opts); + } + + @Override + public synchronized void update(String writer, Map> cells) throws TException { + delegate.update(writer, cells); + } + + @Override + public synchronized void flush(String writer) throws UnknownWriter, MutationsRejectedException, TException { + delegate.flush(writer); + } + + @Override + public synchronized void closeWriter(String writer) throws UnknownWriter, MutationsRejectedException, TException { + delegate.closeWriter(writer); + } + + @Override + public synchronized ConditionalStatus updateRowConditionally(ByteBuffer login, String tableName, ByteBuffer row, ConditionalUpdates updates) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.updateRowConditionally(login, tableName, row, updates); + } + + @Override + public synchronized String createConditionalWriter(ByteBuffer login, String tableName, ConditionalWriterOptions options) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TableNotFoundException, + TException { + return delegate.createConditionalWriter(login, tableName, options); + } + + @Override + public synchronized Map updateRowsConditionally(String conditionalWriter, Map updates) + throws UnknownWriter, org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, TException { + return delegate.updateRowsConditionally(conditionalWriter, updates); + } + + @Override + public synchronized void closeConditionalWriter(String conditionalWriter) throws TException { + delegate.closeConditionalWriter(conditionalWriter); + } + + @Override + public synchronized Range getRowRange(ByteBuffer row) throws TException { + return delegate.getRowRange(row); + } + + @Override + public synchronized Key getFollowing(Key key, PartialKey part) throws TException { + return delegate.getFollowing(key, part); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceError.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceError.java new file mode 100644 index 0000000..3120a6f --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceError.java @@ -0,0 +1,39 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +/** + * Error throws when an unrecoverable condition has been reached. + */ +public class ProxyInstanceError extends Error { + + private static final long serialVersionUID = -6542283312575106269L; + + /** + * Creates a new error with the given cause. + * + * @param cause + * the root cause of this error + */ + public ProxyInstanceError(Throwable cause) { + super(cause); + } + + public ProxyInstanceError(String msg) { + super(msg); + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceOperations.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceOperations.java new file mode 100644 index 0000000..e29bbca --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyInstanceOperations.java @@ -0,0 +1,114 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.admin.ActiveCompaction; +import org.apache.accumulo.core.client.admin.ActiveScan; +import org.apache.accumulo.core.client.admin.InstanceOperations; +import org.apache.thrift.TException; + +/** + * Provides a pass through for InstaceOperations to a proxy server. + */ +class ProxyInstanceOperations implements InstanceOperations { + + ProxyConnector connector; + ByteBuffer token; + + ProxyInstanceOperations(ProxyConnector connector, ByteBuffer token) { + this.connector = connector; + this.token = token; + } + + public void setProperty(String property, String value) throws AccumuloException, AccumuloSecurityException { + try { + connector.getClient().setProperty(token, property, value); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void removeProperty(String property) throws AccumuloException, AccumuloSecurityException { + try { + connector.getClient().removeProperty(token, property); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Map getSystemConfiguration() throws AccumuloException, AccumuloSecurityException { + try { + return connector.getClient().getSystemConfiguration(token); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Map getSiteConfiguration() throws AccumuloException, AccumuloSecurityException { + try { + return connector.getClient().getSiteConfiguration(token); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public List getTabletServers() { + try { + return connector.getClient().getTabletServers(token); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public List getActiveScans(String tserver) throws AccumuloException, AccumuloSecurityException { + try { + return ThriftHelper.fromThriftActiveScans(connector.getClient().getActiveScans(token, tserver)); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public List getActiveCompactions(String tserver) throws AccumuloException, AccumuloSecurityException { + try { + return ThriftHelper.fromThriftActiveCompactions(connector.tableOperations().tableIdMap(), connector.getClient().getActiveCompactions(token, tserver)); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void ping(String tserver) throws AccumuloException { + try { + connector.getClient().pingTabletServer(token, tserver); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public boolean testClassLoad(String className, String asTypeName) throws AccumuloException, AccumuloSecurityException { + try { + return connector.getClient().testClassLoad(token, className, asTypeName); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyMultiTableBatchWriter.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyMultiTableBatchWriter.java new file mode 100644 index 0000000..e353b4b --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyMultiTableBatchWriter.java @@ -0,0 +1,215 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.MultiTableBatchWriter; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.security.SecurityErrorCode; +import org.apache.accumulo.core.data.ConstraintViolationSummary; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.proxy.thrift.UnknownWriter; +import org.apache.accumulo.proxy.thrift.WriterOptions; +import org.apache.thrift.TException; +import org.slf4j.LoggerFactory; + +/** + * Implementation of a MultiTableBatchWriter for the Proxy Server. The ProxyServer API does not actually expose a MultiTableBatchWriter. Instead, we will "fake" + * it by managing our own internal list of "regular" ProxyBatchWriters in this class, one for each table requested. + */ +class ProxyMultiTableBatchWriter implements MultiTableBatchWriter { + + ProxyConnector connector; + ByteBuffer token; + BatchWriterConfig config; + + MutationBuffer buffer; + Map writers; + + ProxyMultiTableBatchWriter(ProxyConnector connector, ByteBuffer token, BatchWriterConfig config) { + this.connector = connector; + this.token = token; + this.config = config; + writers = new HashMap(); + buffer = new MutationBuffer(connector, config); + } + + private void checkClosed() throws IllegalStateException { + if (isClosed()) { + throw new IllegalStateException("Closed."); + } + } + + public BatchWriter getBatchWriter(String table) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + checkClosed(); + + MultiProxyBatchWriter pbw = writers.get(table); + + if (pbw == null) { + try { + pbw = new MultiProxyBatchWriter(connector, token, table, buffer); + writers.put(table, pbw); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + return pbw; + } + + public void flush() throws MutationsRejectedException { + flushOrClose(false); + } + + public void close() throws MutationsRejectedException { + flushOrClose(true); + } + + private void flushOrClose(boolean close) throws MutationsRejectedException { + checkClosed(); + + buffer.flush(); + + List exceptions = null; + try { + try { + for (MultiProxyBatchWriter pbw : writers.values()) { + if (close) { + pbw.multiClose(); + } else { + pbw.multiFlush(); + } + } + } catch (UnknownWriter e) { + throw ExceptionFactory.runtimeException(e); + } catch (org.apache.accumulo.proxy.thrift.MutationsRejectedException e) { + if (exceptions == null) { + exceptions = new ArrayList(); + } + exceptions.add(ThriftHelper.fromThrift(e, connector.getInstance())); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + + writers.clear(); + } finally { + writers = null; + } + + checkExceptions(exceptions); + } + + private void checkExceptions(List mres) throws MutationsRejectedException { + if (mres == null || mres.isEmpty()) { + return; + } + + List cvsList = new LinkedList(); + HashMap> map = new HashMap>(); + Collection serverSideErrors = new LinkedList(); + int unknownErrors = 0; + + for (MutationsRejectedException mre : mres) { + cvsList.addAll(mre.getConstraintViolationSummaries()); + map.putAll(mre.getAuthorizationFailuresMap()); + serverSideErrors.addAll(mre.getErrorServers()); + unknownErrors += mre.getUnknownExceptions(); + } + + throw new MutationsRejectedException(null, cvsList, map, serverSideErrors, unknownErrors, null); + + } + + public boolean isClosed() { + return writers == null; + } + + @Override + protected void finalize() { + if (!isClosed()) { + LoggerFactory.getLogger(ProxyMultiTableBatchWriter.class).warn( + "ProxyMultiTableBatchWriter" + " in finalize but not closed; " + "you forgot to close a MultiTableBatchWriter!"); + + try { + close(); + } catch (MutationsRejectedException mre) { + LoggerFactory.getLogger(ProxyBatchScanner.class).warn("Problem closing ProxyMultiTableBatchWriter.", mre); + } + } + } + + private class MultiProxyBatchWriter implements BatchWriter { + + ProxyConnector connector; + String writerId; + MutationBuffer buffer; + + MultiProxyBatchWriter(ProxyConnector connector, ByteBuffer token, String table, MutationBuffer buffer) + throws org.apache.accumulo.proxy.thrift.AccumuloException, org.apache.accumulo.proxy.thrift.AccumuloSecurityException, + org.apache.accumulo.proxy.thrift.TableNotFoundException, TException { + this.connector = connector; + this.buffer = buffer; + writerId = connector.getClient().createWriter(token, table, new WriterOptions()); + } + + @Override + public void addMutation(Mutation m) throws MutationsRejectedException { + checkClosed(); + + buffer.addMutation(writerId, m); + } + + @Override + public void addMutations(Iterable iterable) throws MutationsRejectedException { + for (Mutation m : iterable) { + addMutation(m); + } + } + + @Override + public void flush() throws MutationsRejectedException { + throw new RuntimeException("Cannot flush BatchWriter created by MultiTableBatchWriter directly; " + "flush MultiTableBatchWriter instead."); + } + + @Override + public void close() throws MutationsRejectedException { + throw new RuntimeException("Cannot close BatchWriter created by MultiTableBatchWriter directly; " + "close MultiTableBatchWriter instead."); + + } + + void multiClose() throws UnknownWriter, org.apache.accumulo.proxy.thrift.MutationsRejectedException, TException { + connector.getClient().closeWriter(writerId); + } + + void multiFlush() throws UnknownWriter, org.apache.accumulo.proxy.thrift.MutationsRejectedException, TException { + connector.getClient().flush(writerId); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyScanner.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyScanner.java new file mode 100644 index 0000000..da39e82 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyScanner.java @@ -0,0 +1,172 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; + +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.proxy.thrift.ScanColumn; +import org.apache.accumulo.proxy.thrift.ScanOptions; +import org.apache.thrift.TException; +import org.slf4j.LoggerFactory; + +class ProxyScanner extends AbstractProxyScanner implements Scanner { + + protected ScanOptions options; + int batchSize = 50; + + ProxyScanner(ProxyConnector connector, ByteBuffer token, String tableName, Authorizations auths) { + super(connector, token, tableName); + options = new ScanOptions(); + for (ByteBuffer auth : auths.getAuthorizationsBB()) { + options.addToAuthorizations(auth); + } + } + + public void addScanIterator(IteratorSetting cfg) { + options.addToIterators(ThriftHelper.toThrift(cfg)); + } + + public void removeScanIterator(String iteratorName) { + for (org.apache.accumulo.proxy.thrift.IteratorSetting is : options.iterators) { + if (is.getName().equals(iteratorName)) { + options.iterators.remove(is); + break; + } + } + } + + public void updateScanIteratorOption(String iteratorName, String key, String value) { + for (org.apache.accumulo.proxy.thrift.IteratorSetting is : options.iterators) { + if (is.getName().equals(iteratorName)) { + is.putToProperties(key, value); + } + } + } + + @Override + protected void addToFetchOptions(ScanColumn col) { + options.addToColumns(col); + } + + public void clearColumns() { + if (options.getColumns() != null) { + options.getColumns().clear(); + } + } + + public void clearScanIterators() { + if (options.getIterators() != null) { + options.getIterators().clear(); + } + } + + public Iterator> iterator() { + // TODO Close if older instance open? + try { + scannerId = connector.getClient().createScanner(token, tableName, options); + + // TODO Someone needs to close this scannerID! + return new ScannerIterator(scannerId, connector, batchSize); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public void setTimeout(long timeOut, TimeUnit timeUnit) { + throw ExceptionFactory.unsupported(); + } + + public long getTimeout(TimeUnit timeUnit) { + throw ExceptionFactory.unsupported(); + } + + public void close() { + try { + connector.getClient().closeScanner(scannerId); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } finally { + scannerId = null; + } + } + + @Override + protected void finalize() { + if (scannerId != null) { + close(); + LoggerFactory.getLogger(ProxyBatchScanner.class).warn("Scanner " + scannerId + " in finalize but not closed; " + "you forgot to close a scanner!"); + } + } + + public void setRange(Range range) { + if (range == null) { + options.unsetRange(); + } else { + options.setRange(ThriftHelper.toThrift(range)); + } + } + + public Range getRange() { + return ThriftHelper.fromThrift(options.getRange()); + } + + public void setBatchSize(int size) { + if (size < 1) { + throw new IllegalArgumentException("Batch size must be > 0; provided " + size); + } + this.batchSize = size; + } + + public int getBatchSize() { + return batchSize; + } + + public void enableIsolation() { + throw ExceptionFactory.unsupported(); + } + + public void disableIsolation() { + throw ExceptionFactory.unsupported(); + } + + public long getReadaheadThreshold() { + throw ExceptionFactory.unsupported(); + } + + public void setReadaheadThreshold(long batches) { + throw ExceptionFactory.unsupported(); + } + + @Deprecated + public void setTimeOut(int timeOut) { + setTimeout(timeOut, TimeUnit.SECONDS); + } + + @Deprecated + public int getTimeOut() { + return (int) this.getTimeout(TimeUnit.SECONDS); + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxySecurityOperations.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxySecurityOperations.java new file mode 100644 index 0000000..94795bd --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxySecurityOperations.java @@ -0,0 +1,209 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import static edu.jhuapl.accumulo.proxy.ThriftHelper.UTF8; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.admin.SecurityOperations; +import org.apache.accumulo.core.client.security.tokens.AuthenticationToken; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.security.NamespacePermission; +import org.apache.accumulo.core.security.SystemPermission; +import org.apache.accumulo.core.security.TablePermission; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.thrift.TException; + +/** + * Implements SecurityOperations as a pass through to a proxy server. + */ + +class ProxySecurityOperations implements SecurityOperations { + + AccumuloProxy.Iface client; + ByteBuffer token; + + ProxySecurityOperations(ProxyConnector connector, ByteBuffer token) { + this.client = connector.getClient(); + this.token = token; + } + + @Deprecated + public void createUser(String user, byte[] password, Authorizations authorizations) throws AccumuloException, AccumuloSecurityException { + createLocalUser(user, new PasswordToken(password)); + changeUserAuthorizations(user, authorizations); + } + + public void createLocalUser(String principal, PasswordToken password) throws AccumuloException, AccumuloSecurityException { + try { + client.createLocalUser(token, principal, ByteBuffer.wrap(password.getPassword())); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + @Deprecated + public void dropUser(String user) throws AccumuloException, AccumuloSecurityException { + dropLocalUser(user); + } + + public void dropLocalUser(String principal) throws AccumuloException, AccumuloSecurityException { + try { + client.dropLocalUser(token, principal); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + @Deprecated + public boolean authenticateUser(String user, byte[] password) throws AccumuloException, AccumuloSecurityException { + return this.authenticateUser(user, new PasswordToken(password)); + } + + public boolean authenticateUser(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException { + + if (!(token instanceof PasswordToken)) { + throw ExceptionFactory.notYetImplemented(); + } + PasswordToken passwd = (PasswordToken) token; + try { + Map properties = new HashMap(); + properties.put("password", new String(passwd.getPassword(), UTF8)); + return client.authenticateUser(this.token, principal, properties); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + @Deprecated + public void changeUserPassword(String user, byte[] password) throws AccumuloException, AccumuloSecurityException { + changeLocalUserPassword(user, new PasswordToken(password)); + } + + public void changeLocalUserPassword(String principal, PasswordToken token) throws AccumuloException, AccumuloSecurityException { + try { + client.changeLocalUserPassword(this.token, principal, ByteBuffer.wrap(token.getPassword())); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void changeUserAuthorizations(String principal, Authorizations authorizations) throws AccumuloException, AccumuloSecurityException { + try { + client.changeUserAuthorizations(token, principal, new HashSet(authorizations.getAuthorizationsBB())); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Authorizations getUserAuthorizations(String principal) throws AccumuloException, AccumuloSecurityException { + try { + return new Authorizations(client.getUserAuthorizations(token, principal)); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + /** + * ProxyAPI specifies AccumuloException and AccumuloSecurityException may be thrown. The IllegalArgumentException may be thrown if the + * {@link ThriftHelper#convertEnum(Enum, Class)} fails because the Java and Thrift SystemPermission no longer match; this should never happen. + */ + public boolean hasSystemPermission(String principal, SystemPermission perm) throws AccumuloException, AccumuloSecurityException, InternalError { + try { + return client.hasSystemPermission(token, principal, ThriftHelper.convertEnum(perm, org.apache.accumulo.proxy.thrift.SystemPermission.class)); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public boolean hasTablePermission(String principal, String table, TablePermission perm) throws AccumuloException, AccumuloSecurityException { + try { + return client.hasTablePermission(token, principal, table, ThriftHelper.convertEnum(perm, org.apache.accumulo.proxy.thrift.TablePermission.class)); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public boolean hasNamespacePermission(String principal, String namespace, NamespacePermission perm) throws AccumuloException, AccumuloSecurityException { + throw ExceptionFactory.unsupported(); + } + + public void grantSystemPermission(String principal, SystemPermission permission) throws AccumuloException, AccumuloSecurityException { + try { + client.grantSystemPermission(token, principal, ThriftHelper.convertEnum(permission, org.apache.accumulo.proxy.thrift.SystemPermission.class)); + + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void grantTablePermission(String principal, String table, TablePermission permission) throws AccumuloException, AccumuloSecurityException { + try { + client.grantTablePermission(token, principal, table, ThriftHelper.convertEnum(permission, org.apache.accumulo.proxy.thrift.TablePermission.class)); + + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void grantNamespacePermission(String principal, String namespace, NamespacePermission permission) throws AccumuloException, AccumuloSecurityException { + throw ExceptionFactory.unsupported(); + } + + public void revokeSystemPermission(String principal, SystemPermission permission) throws AccumuloException, AccumuloSecurityException { + try { + client.revokeSystemPermission(token, principal, ThriftHelper.convertEnum(permission, org.apache.accumulo.proxy.thrift.SystemPermission.class)); + + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void revokeTablePermission(String principal, String table, TablePermission permission) throws AccumuloException, AccumuloSecurityException { + try { + client.revokeTablePermission(token, principal, table, ThriftHelper.convertEnum(permission, org.apache.accumulo.proxy.thrift.TablePermission.class)); + + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void revokeNamespacePermission(String principal, String namespace, NamespacePermission permission) throws AccumuloException, AccumuloSecurityException { + throw ExceptionFactory.unsupported(); + } + + @Deprecated + public Set listUsers() throws AccumuloException, AccumuloSecurityException { + return listLocalUsers(); + } + + public Set listLocalUsers() throws AccumuloException, AccumuloSecurityException { + try { + return client.listLocalUsers(token); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyTableOperations.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyTableOperations.java new file mode 100644 index 0000000..dfbac89 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ProxyTableOperations.java @@ -0,0 +1,528 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.DiskUsage; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.admin.TimeType; +import org.apache.accumulo.core.client.impl.thrift.SecurityErrorCode; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.proxy.thrift.AccumuloProxy; +import org.apache.hadoop.io.Text; +import org.apache.thrift.TException; + +/** + * Implements TableOperations as a pass through to a proxy server. + */ +class ProxyTableOperations implements TableOperations { + + ProxyConnector connector; + AccumuloProxy.Iface client; + ByteBuffer token; + + ProxyTableOperations(ProxyConnector connector, ByteBuffer token) { + this.connector = connector; + this.client = connector.getClient(); + this.token = token; + } + + public SortedSet list() { + try { + return new TreeSet(client.listTables(token)); + } catch (TException te) { + throw ExceptionFactory.runtimeException(te); + } + } + + public boolean exists(String tableName) { + return list().contains(tableName); + } + + public void create(String tableName) throws AccumuloException, AccumuloSecurityException, TableExistsException { + create(tableName, true); + } + + public void create(String tableName, boolean limitVersion) throws AccumuloException, AccumuloSecurityException, TableExistsException { + create(tableName, limitVersion, TimeType.MILLIS); + } + + public void create(String tableName, boolean versioningIter, TimeType timeType) throws AccumuloException, AccumuloSecurityException, TableExistsException { + + org.apache.accumulo.proxy.thrift.TimeType ttype = org.apache.accumulo.proxy.thrift.TimeType.valueOf(timeType.toString()); + try { + client.createTable(token, tableName, versioningIter, ttype); + } catch (org.apache.accumulo.proxy.thrift.TableExistsException tee) { + throw ExceptionFactory.tableExistsException(tableName, tee); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void importTable(String tableName, String importDir) throws TableExistsException, AccumuloException, AccumuloSecurityException { + try { + client.importTable(token, tableName, importDir); + } catch (org.apache.accumulo.proxy.thrift.TableExistsException tee) { + throw ExceptionFactory.tableExistsException(tableName, tee); + } catch (TException te) { + throw ExceptionFactory.accumuloException(te); + } + } + + public void exportTable(String tableName, String exportDir) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + try { + client.exportTable(token, tableName, exportDir); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException te) { + throw ExceptionFactory.accumuloException(te); + } + } + + public void addSplits(String tableName, SortedSet partitionKeys) throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + Set tsplits = new TreeSet(); + for (Text key : partitionKeys) { + tsplits.add(ByteBuffer.wrap(key.getBytes())); + } + + try { + client.addSplits(token, tableName, tsplits); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException te) { + throw ExceptionFactory.accumuloException(te); + } + } + + @Deprecated + public Collection getSplits(String tableName) throws TableNotFoundException { + try { + return listSplits(tableName); + } catch (AccumuloSecurityException e) { + throw ExceptionFactory.runtimeException(e); + } catch (AccumuloException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public Collection listSplits(String tableName) throws TableNotFoundException, AccumuloSecurityException, AccumuloException { + return listSplits(tableName, Integer.MAX_VALUE); + } + + @Deprecated + public Collection getSplits(String tableName, int maxSplits) throws TableNotFoundException { + try { + return listSplits(tableName, maxSplits); + } catch (AccumuloSecurityException e) { + throw ExceptionFactory.runtimeException(e); + } catch (AccumuloException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public Collection listSplits(String tableName, int maxSplits) throws TableNotFoundException, AccumuloSecurityException, AccumuloException { + try { + List list = client.listSplits(token, tableName, maxSplits); + + List text = new ArrayList(list.size()); + for (ByteBuffer bb : list) { + text.add(new Text(bb.array())); + } + return text; + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException te) { + throw ExceptionFactory.accumuloException(te); + } + } + + public Text getMaxRow(String tableName, Authorizations auths, Text startRow, boolean startInclusive, Text endRow, boolean endInclusive) + throws TableNotFoundException, AccumuloException, AccumuloSecurityException { + try { + ByteBuffer answer = client.getMaxRow(token, tableName, new HashSet(auths.getAuthorizationsBB()), ThriftHelper.toByteBuffer(startRow), + startInclusive, ThriftHelper.toByteBuffer(endRow), endInclusive); + return new Text(answer.array()); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + client.mergeTablets(token, tableName, ThriftHelper.toByteBuffer(start), ThriftHelper.toByteBuffer(end)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void deleteRows(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + client.deleteRows(token, tableName, ThriftHelper.toByteBuffer(start), ThriftHelper.toByteBuffer(end)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void compact(String tableName, Text start, Text end, boolean flush, boolean wait) throws AccumuloSecurityException, TableNotFoundException, + AccumuloException { + compact(tableName, start, end, null, flush, wait); + } + + public void compact(String tableName, Text start, Text end, List iterators, boolean flush, boolean wait) throws AccumuloSecurityException, + TableNotFoundException, AccumuloException { + try { + client.compactTable(token, tableName, ThriftHelper.toByteBuffer(start), ThriftHelper.toByteBuffer(end), ThriftHelper.toThriftIteratorSettings(iterators), + flush, wait); + } catch (org.apache.accumulo.proxy.thrift.AccumuloSecurityException e) { + throw new AccumuloSecurityException(connector.whoami(), SecurityErrorCode.DEFAULT_SECURITY_ERROR, "", e); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (org.apache.accumulo.proxy.thrift.AccumuloException e) { + throw ExceptionFactory.accumuloException(e); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void cancelCompaction(String tableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException { + try { + client.cancelCompaction(token, tableName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void delete(String tableName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + client.deleteTable(token, tableName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void clone(String srcTableName, String newTableName, boolean flush, Map propertiesToSet, Set propertiesToExclude) + throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { + try { + client.cloneTable(token, srcTableName, newTableName, flush, propertiesToSet, propertiesToExclude); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(srcTableName, tnfe); + } catch (org.apache.accumulo.proxy.thrift.TableExistsException tee) { + throw ExceptionFactory.tableExistsException(newTableName, tee); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void rename(String oldTableName, String newTableName) throws AccumuloSecurityException, TableNotFoundException, AccumuloException, + TableExistsException { + try { + client.renameTable(token, oldTableName, newTableName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(oldTableName, tnfe); + } catch (org.apache.accumulo.proxy.thrift.TableExistsException tee) { + throw ExceptionFactory.tableExistsException(newTableName, tee); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + @Deprecated + public void flush(String tableName) throws AccumuloException, AccumuloSecurityException { + try { + flush(tableName, null, null, true); + } catch (TableNotFoundException tnfe) { + throw ExceptionFactory.accumuloException(tnfe); + } + } + + public void flush(String tableName, Text start, Text end, boolean wait) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + client.flushTable(token, tableName, ThriftHelper.toByteBuffer(start), ThriftHelper.toByteBuffer(end), wait); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void setProperty(String tableName, String property, String value) throws AccumuloException, AccumuloSecurityException { + try { + client.setProperty(token, property, value); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void removeProperty(String tableName, String property) throws AccumuloException, AccumuloSecurityException { + try { + client.removeProperty(token, property); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Iterable> getProperties(String tableName) throws AccumuloException, TableNotFoundException { + try { + return client.getTableProperties(token, tableName).entrySet(); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void setLocalityGroups(String tableName, Map> groups) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + + try { + Map> tgroups = new HashMap>(); + for (Entry> entry : groups.entrySet()) { + Set tset = new HashSet(); + tgroups.put(entry.getKey(), tset); + for (Text t : entry.getValue()) { + tset.add(t.toString()); + } + } + client.setLocalityGroups(token, tableName, tgroups); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Map> getLocalityGroups(String tableName) throws AccumuloException, TableNotFoundException { + try { + Map> tgroups = client.getLocalityGroups(token, tableName); + Map> groups = new HashMap>(); + for (Entry> tentry : tgroups.entrySet()) { + Set set = new HashSet(); + groups.put(tentry.getKey(), set); + for (String t : tentry.getValue()) { + set.add(new Text(t)); + } + } + return groups; + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Set splitRangeByTablets(String tableName, Range range, int maxSplits) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException { + try { + return ThriftHelper.fromThriftRanges(client.splitRangeByTablets(token, tableName, ThriftHelper.toThrift(range), maxSplits)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void importDirectory(String tableName, String dir, String failureDir, boolean setTime) throws TableNotFoundException, IOException, AccumuloException, + AccumuloSecurityException { + try { + client.importDirectory(token, tableName, dir, failureDir, setTime); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void offline(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + offline(tableName, true); + } + + public void offline(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + try { + client.offlineTable(token, tableName, wait); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void online(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + online(tableName, true); + } + + public void online(String tableName, boolean wait) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + try { + client.onlineTable(token, tableName, wait); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + + } + + public void clearLocatorCache(String tableName) throws TableNotFoundException { + try { + client.clearLocatorCache(token, tableName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public Map tableIdMap() { + try { + return client.tableIdMap(token); + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public void attachIterator(String tableName, IteratorSetting setting) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + attachIterator(tableName, setting, null); + } + + public void attachIterator(String tableName, IteratorSetting setting, EnumSet scopes) throws AccumuloSecurityException, AccumuloException, + TableNotFoundException { + try { + client.attachIterator(token, tableName, ThriftHelper.toThrift(setting), ThriftHelper.toThrift(scopes)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void removeIterator(String tableName, String name, EnumSet scopes) throws AccumuloSecurityException, AccumuloException, + TableNotFoundException { + try { + client.removeIterator(token, tableName, name, ThriftHelper.toThrift(scopes)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public IteratorSetting getIteratorSetting(String tableName, String name, IteratorScope scope) throws AccumuloSecurityException, AccumuloException, + TableNotFoundException { + try { + return ThriftHelper.fromThrift(client.getIteratorSetting(token, tableName, name, ThriftHelper.toThrift(scope))); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Map> listIterators(String tableName) throws AccumuloSecurityException, AccumuloException, TableNotFoundException { + try { + return ThriftHelper.fromThrift(client.listIterators(token, tableName)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void checkIteratorConflicts(String tableName, IteratorSetting setting, EnumSet scopes) throws AccumuloException, TableNotFoundException { + try { + client.checkIteratorConflicts(token, tableName, ThriftHelper.toThrift(setting), ThriftHelper.toThrift(scopes)); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public int addConstraint(String tableName, String constraintClassName) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + return client.addConstraint(token, tableName, constraintClassName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public void removeConstraint(String tableName, int constraint) throws AccumuloException, AccumuloSecurityException { + try { + client.removeConstraint(token, tableName, constraint); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public Map listConstraints(String tableName) throws AccumuloException, TableNotFoundException { + try { + return client.listConstraints(token, tableName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public List getDiskUsage(Set tables) throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + try { + return new ArrayList(ThriftHelper.fromThriftDiskUsage(client.getDiskUsage(token, tables))); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + + public boolean testClassLoad(String tableName, String className, String asTypeName) throws AccumuloException, AccumuloSecurityException, + TableNotFoundException { + try { + return client.testClassLoad(token, className, asTypeName); + } catch (org.apache.accumulo.proxy.thrift.TableNotFoundException tnfe) { + throw ExceptionFactory.tableNotFoundException(tableName, tnfe); + } catch (TException e) { + throw ExceptionFactory.accumuloException(e); + } + } + +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ScannerIterator.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ScannerIterator.java new file mode 100644 index 0000000..d89cc23 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ScannerIterator.java @@ -0,0 +1,99 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.NoSuchElementException; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyValue; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.proxy.thrift.ScanResult; +import org.apache.thrift.TException; + +/** + * Givens a scanner id and connection to a proxy server, creates an iterator for the KV provided for the scanner id from the proxy. + */ +class ScannerIterator implements Iterator> { + + String id; + ProxyConnector connector; + int fetchSize; + + private boolean hasMore; + private Iterator results; + + ScannerIterator(String id, ProxyConnector connector, int fetchSize) { + this.id = id; + this.connector = connector; + this.fetchSize = fetchSize; + + // setup to make the class look for "more" data on 1st pass... + this.results = Collections.emptyListIterator(); + this.hasMore = true; + } + + public boolean hasNext() { + try { + while (!results.hasNext()) { + // either first pass or we are at the end of the current batch. + if (!hasMore) { + // last check told us we were done at the end of this; + // hasMore is initialized to true to ensure this does not + // fail on 1st call + return false; + } + + ScanResult sr = connector.getClient().nextK(id, fetchSize); + results = sr.getResultsIterator(); + hasMore = sr.isMore(); + + // We cannot return true from this method unless there is at + // least one result in the results list. There is a possibility + // that the results are empty now, but hasMore is still true. + // Therefore, we iterate until either hasMore is false (we are + // done!) or results.hasNext() is true + } + + // if we got here, there is something in results + return true; + } catch (TException e) { + throw ExceptionFactory.runtimeException(e); + } + } + + public Entry next() { + if (!hasNext()) { + // need to call hasNext to make sure results is setup correctly; + // might as well use the return to throw our own exception here + // instead of waiting for the call to results.next() to generate + // it... + throw new NoSuchElementException(); + } + + org.apache.accumulo.proxy.thrift.KeyValue kv = results.next(); + org.apache.accumulo.proxy.thrift.Key k = kv.getKey(); + + Key key = new Key(k.getRow(), k.getColFamily(), k.getColQualifier(), k.getColVisibility(), k.getTimestamp()); + return new KeyValue(key, kv.getValue()); + } + + public void remove() { + throw new UnsupportedOperationException(); + } +} diff --git a/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ThriftHelper.java b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ThriftHelper.java new file mode 100644 index 0000000..4372d51 --- /dev/null +++ b/proxy-instance/src/main/java/edu/jhuapl/accumulo/proxy/ThriftHelper.java @@ -0,0 +1,489 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.admin.ActiveCompaction; +import org.apache.accumulo.core.client.admin.ActiveCompaction.CompactionReason; +import org.apache.accumulo.core.client.admin.ActiveCompaction.CompactionType; +import org.apache.accumulo.core.client.admin.ActiveScan; +import org.apache.accumulo.core.client.admin.DiskUsage; +import org.apache.accumulo.core.client.admin.ScanState; +import org.apache.accumulo.core.client.admin.ScanType; +import org.apache.accumulo.core.data.Column; +import org.apache.accumulo.core.data.ColumnUpdate; +import org.apache.accumulo.core.data.Condition; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.KeyExtent; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope; +import org.apache.hadoop.io.Text; + +/** + * Utility class to translate between Accumulo<->Thrift objects. + * + */ +final class ThriftHelper { + + /** + * UTF-8 charset that can be used to convert String to/from byte arrays. + */ + static final Charset UTF8 = Charset.forName("UTF-8"); + + /** + * As a utility should not be instantiated. + */ + private ThriftHelper() { + + } + + public static Map> fromThrift(Map> tmap) { + if (tmap == null) { + return null; + } + + Map> map = new HashMap>(); + + for (Entry> entry : tmap.entrySet()) { + map.put(entry.getKey(), fromThrift(entry.getValue())); + } + return map; + } + + public static Set toThrift(EnumSet scopes) { + if (scopes == null) { + return null; + } + + HashSet set = new HashSet(); + for (IteratorScope is : scopes) { + set.add(toThrift(is)); + } + return set; + } + + public static EnumSet fromThrift(Set tscopes) { + if (tscopes == null) { + return null; + } + + LinkedList list = new LinkedList(); + for (org.apache.accumulo.proxy.thrift.IteratorScope tis : tscopes) { + list.add(fromThrift(tis)); + } + + return EnumSet.copyOf(list); + } + + public static org.apache.accumulo.proxy.thrift.IteratorScope toThrift(IteratorScope is) { + // we can't just use the enum values, since thrift MINC=0 but Apache + // minc=1. we can't just use the names, since thrift are in upper case, + // and/ Apache are lower. assuming Thrift will remain uppercase and use + // the same characters as Apache + try { + return org.apache.accumulo.proxy.thrift.IteratorScope.valueOf(is.name().toUpperCase()); + } catch (IllegalArgumentException ex) { + throw ExceptionFactory.noEnumMapping(is, org.apache.accumulo.proxy.thrift.IteratorScope.class); + } + } + + public static IteratorScope fromThrift(org.apache.accumulo.proxy.thrift.IteratorScope tis) { + if (tis == null) { + return null; + } + + // we can't just use the enum values, since thrift MINC=0 but Apache + // minc=1. we can't just use the names, since thrift are in upper case, + // and Apache are lower. assuming Thrift will remain uppercase and use + // the same characters as Apache + try { + return IteratorScope.valueOf(tis.name().toLowerCase()); + } catch (IllegalArgumentException ex) { + throw ExceptionFactory.noEnumMapping(tis, IteratorScope.class); + } + } + + public static org.apache.accumulo.proxy.thrift.IteratorSetting toThrift(IteratorSetting is) { + if (is == null) { + return null; + } + + org.apache.accumulo.proxy.thrift.IteratorSetting tis = new org.apache.accumulo.proxy.thrift.IteratorSetting(); + tis.setIteratorClass(is.getIteratorClass()); + tis.setName(is.getName()); + tis.setPriority(is.getPriority()); + tis.setProperties(is.getOptions()); + return tis; + } + + public static org.apache.accumulo.proxy.thrift.Key toThrift(Key key) { + if (key == null) { + return null; + } + + org.apache.accumulo.proxy.thrift.Key tkey = new org.apache.accumulo.proxy.thrift.Key(); + tkey.setRow(key.getRow().getBytes()); + tkey.setColFamily(key.getColumnFamily().getBytes()); + tkey.setColQualifier(key.getColumnQualifier().getBytes()); + tkey.setColVisibility(key.getColumnVisibility().getBytes()); + tkey.setTimestamp(key.getTimestamp()); + return tkey; + } + + public static Key fromThrift(org.apache.accumulo.proxy.thrift.Key tkey) { + if (tkey == null) { + return null; + } + + return new Key(tkey.getRow(), tkey.getColFamily(), tkey.getColQualifier(), tkey.getColVisibility(), tkey.getTimestamp()); + } + + public static List toThriftRanges(Collection ranges) { + if (ranges == null) { + return null; + } + + List tranges = new ArrayList(ranges.size()); + for (Range range : ranges) { + tranges.add(toThrift(range)); + } + return tranges; + } + + public static org.apache.accumulo.proxy.thrift.Range toThrift(Range range) { + if (range == null) { + return null; + } + + org.apache.accumulo.proxy.thrift.Range trange = new org.apache.accumulo.proxy.thrift.Range(); + trange.setStart(toThrift(range.getStartKey())); + trange.setStop(toThrift(range.getEndKey())); + trange.setStartInclusive(range.isStartKeyInclusive()); + trange.setStopInclusive(range.isEndKeyInclusive()); + return trange; + } + + public static Set fromThriftRanges(Set tranges) { + if (tranges == null) { + return null; + } + + Set ranges = new HashSet(); + for (org.apache.accumulo.proxy.thrift.Range trange : tranges) { + ranges.add(fromThrift(trange)); + } + return ranges; + } + + public static Range fromThrift(org.apache.accumulo.proxy.thrift.Range trange) { + if (trange == null) { + return null; + } + + return new Range(fromThrift(trange.getStart()), trange.isStartInclusive(), fromThrift(trange.getStop()), trange.isStopInclusive()); + } + + /** + * Converts each non-thrift ColumnUpdate in {@code list} to a Thrift ColumnUpdate (see {@link #toThrift(ColumnUpdate)}) and adds it to the provided + * {@code thriftUpdates} list. If {@code updates} is null or empty, {@code thriftUpdates} will not be changed. + * + * @param thriftUpdates + * the list where the newly created Thrift ColumnUpdates will be added. + * @param updates + * the list of ColumnUpdates to be converted to Thrift ColumnUpdates + */ + public static void addThriftColumnUpdates(List thriftUpdates, List updates) { + if (updates == null) { + return; + } + + for (ColumnUpdate cu : updates) { + thriftUpdates.add(toThrift(cu)); + } + } + + public static org.apache.accumulo.proxy.thrift.ColumnUpdate toThrift(ColumnUpdate update) { + if (update == null) { + return null; + } + + org.apache.accumulo.proxy.thrift.ColumnUpdate tcu = new org.apache.accumulo.proxy.thrift.ColumnUpdate(); + tcu.setColFamily(update.getColumnFamily()); + tcu.setColQualifier(update.getColumnQualifier()); + tcu.setColVisibility(update.getColumnVisibility()); + tcu.setTimestamp(update.getTimestamp()); + tcu.setValue(update.getValue()); + // Work around for ACCUMULO-3474 + if (update.isDeleted()) { + tcu.setDeleteCell(update.isDeleted()); + } + return tcu; + } + + public static List fromThriftActiveScans(List tscans) { + if (tscans == null) { + return null; + } + + List scans = new ArrayList(tscans.size()); + for (org.apache.accumulo.proxy.thrift.ActiveScan tscan : tscans) { + scans.add(fromThrift(tscan)); + } + return scans; + } + + public static ActiveScan fromThrift(org.apache.accumulo.proxy.thrift.ActiveScan scan) { + if (scan == null) { + return null; + } + + // Note- scanId is not provided by the server and does not appear to be + // set/used in actual implementation... we will just set it to 0. + return new ProxyActiveScan(0, scan.getClient(), scan.getUser(), scan.getTable(), scan.getAge(), System.currentTimeMillis() - scan.getIdleTime(), + convertEnum(scan.getType(), ScanType.class), convertEnum(scan.getState(), ScanState.class), fromThrift(scan.getExtent()), + fromThriftColumns(scan.getColumns()), scan.getAuthorizations(), scan.getIdleTime()); + + } + + public static KeyExtent fromThrift(org.apache.accumulo.proxy.thrift.KeyExtent textent) { + if (textent == null) { + return null; + } + + return new KeyExtent(toText(textent.getTableId()), toText(textent.getEndRow()), toText(textent.getPrevEndRow())); + } + + public static List fromThriftColumns(List tcolumns) { + if (tcolumns == null) { + return null; + } + + List columns = new ArrayList(tcolumns.size()); + for (org.apache.accumulo.proxy.thrift.Column tcolumn : tcolumns) { + columns.add(fromThrift(tcolumn)); + } + return columns; + } + + public static Column fromThrift(org.apache.accumulo.proxy.thrift.Column tcolumn) { + if (tcolumn == null) { + return null; + } + + return new Column(tcolumn.getColFamily(), tcolumn.getColQualifier(), tcolumn.getColVisibility()); + } + + public static List fromThriftActiveCompactions(Map tableIdMap, + List tcompactions) { + if (tcompactions == null) { + return null; + } + + List compactions = new ArrayList(tcompactions.size()); + for (org.apache.accumulo.proxy.thrift.ActiveCompaction tcompaction : tcompactions) { + compactions.add(fromThrift(getTableNameFromId(tableIdMap, tcompaction.getExtent().getTableId()), tcompaction)); + } + return compactions; + } + + public static ActiveCompaction fromThrift(String tableName, org.apache.accumulo.proxy.thrift.ActiveCompaction tcompaction) { + if (tcompaction == null) { + return null; + } + + return new ProxyActiveCompaction(tableName, fromThrift(tcompaction.getExtent()), tcompaction.getAge(), tcompaction.getInputFiles(), + tcompaction.getOutputFile(), convertEnum(tcompaction.getType(), CompactionType.class), convertEnum(tcompaction.getReason(), CompactionReason.class), + tcompaction.getLocalityGroup(), tcompaction.getEntriesRead(), tcompaction.getEntriesWritten(), fromThriftIteratorSettings(tcompaction.getIterators())); + } + + public static String getTableNameFromId(Map tableIdMap, String id) { + if (id == null) { + return null; + } + + for (Entry entry : tableIdMap.entrySet()) { + if (id.equalsIgnoreCase(entry.getKey())) { + return entry.getValue(); + } + } + return null; + } + + public static List fromThriftIteratorSettings(List tsettings) { + if (tsettings == null) { + return null; + } + + List settings = new ArrayList(tsettings.size()); + for (org.apache.accumulo.proxy.thrift.IteratorSetting tsetting : tsettings) { + settings.add(fromThrift(tsetting)); + } + return settings; + } + + public static IteratorSetting fromThrift(org.apache.accumulo.proxy.thrift.IteratorSetting tsetting) { + if (tsetting == null) { + return null; + } + + return new IteratorSetting(tsetting.getPriority(), tsetting.getName(), tsetting.getIteratorClass(), tsetting.getProperties()); + } + + public static List toThriftIteratorSettings(List settings) { + if (settings == null) { + return null; + } + + List tsettings = new ArrayList(settings.size()); + for (IteratorSetting setting : settings) { + tsettings.add(toThrift(setting)); + } + return tsettings; + } + + public static List fromThriftDiskUsage(List tusage) { + if (tusage == null) { + return null; + } + + List usage = new ArrayList(tusage.size()); + for (org.apache.accumulo.proxy.thrift.DiskUsage tdu : tusage) { + usage.add(fromThrift(tdu)); + } + return usage; + } + + public static DiskUsage fromThrift(org.apache.accumulo.proxy.thrift.DiskUsage tusage) { + if (tusage == null) { + return null; + } + + return new DiskUsage(new TreeSet(tusage.getTables()), tusage.getUsage()); + } + + public static List toThriftCondition(List conditions) { + if (conditions == null) { + return null; + } + + List tconditions = new ArrayList(conditions.size()); + for (Condition c : conditions) { + tconditions.add(toThrift(c)); + } + return tconditions; + } + + public static org.apache.accumulo.proxy.thrift.Condition toThrift(Condition condition) { + if (condition == null) { + return null; + } + + org.apache.accumulo.proxy.thrift.Condition tcondition = new org.apache.accumulo.proxy.thrift.Condition(); + + org.apache.accumulo.proxy.thrift.Column col = new org.apache.accumulo.proxy.thrift.Column(); + if (condition.getFamily() != null) { + col.setColFamily(condition.getFamily().getBackingArray()); + } + if (condition.getQualifier() != null) { + col.setColQualifier(condition.getQualifier().getBackingArray()); + } + + if (col.getColVisibility() != null) { + col.setColVisibility(condition.getQualifier().getBackingArray()); + } + tcondition.setColumn(col); + + if (condition.getTimestamp() != null) { + tcondition.setTimestamp(condition.getTimestamp()); + } + + if (condition.getValue() != null) { + tcondition.setValue(condition.getValue().getBackingArray()); + } + tcondition.setIterators(toThriftIteratorSettings(Arrays.asList(condition.getIterators()))); + return tcondition; + } + + public static MutationsRejectedException fromThrift(org.apache.accumulo.proxy.thrift.MutationsRejectedException mre, Instance instance) { + return new MutationsRejectedException(instance, null, null, Collections.singleton(mre.getMsg()), 0, mre); + } + + /** + * Converts the enumeration of class F to the enumeration of class S assuming both F and S define the same values. This is used in mapping between Accumulo + * client-side enumerations and their equivalent Thrift-based enumerations. + * + * @param first + * the enumeration to convert + * @param cls + * the new class of the enumeration + * @return an enumeration of type cls with the same name as first. + * @exception ProxyInstanceError + * thrown if the text of first is not a valid enumerated value in the class cls. + */ + public static ,S extends Enum> S convertEnum(F first, Class cls) throws ProxyInstanceError { + if (first == null) { + return null; + } + + try { + return Enum.valueOf(cls, first.toString()); + } catch (IllegalArgumentException e) { + throw ExceptionFactory.noEnumMapping(first, cls); + } + } + + public static Text toText(String val) { + if (val == null) { + return null; + } else { + return new Text(val.getBytes(UTF8)); + } + } + + public static Text toText(byte[] val) { + if (val == null) { + return null; + } else { + return new Text(val); + } + } + + public static ByteBuffer toByteBuffer(Text val) { + if (val == null) { + return null; + } else { + return ByteBuffer.wrap(val.getBytes()); + } + } +} diff --git a/proxy-instance/src/main/resources/Eclipse-Codestyle.xml b/proxy-instance/src/main/resources/Eclipse-Codestyle.xml new file mode 100644 index 0000000..50cf49f --- /dev/null +++ b/proxy-instance/src/main/resources/Eclipse-Codestyle.xml @@ -0,0 +1,292 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/proxy-instance/src/main/resources/license.txt b/proxy-instance/src/main/resources/license.txt new file mode 100644 index 0000000..4c4a47d --- /dev/null +++ b/proxy-instance/src/main/resources/license.txt @@ -0,0 +1,13 @@ +Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ConnectorBase.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ConnectorBase.java new file mode 100644 index 0000000..a74b0f5 --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ConnectorBase.java @@ -0,0 +1,116 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.proxy.Proxy; +import org.apache.thrift.transport.TTransportException; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; + +/** + * Base class for unit and integration tests that want to get access to an Instance and/or Connector. This class provides the means to setup a ProxyInstace + * connection to a local ProxyServer backed by a mock (in-memory) Accumulo instance for unit tests, or to create a ProxyInstace connection to a real (external) + * ProxyServer for integration tests. + */ +public class ConnectorBase { + + protected static volatile ProxyInstance instance = null; + protected static volatile Connector connector = null; + private static Thread proxyServerThread = null; + + /** + * Initializes the instance and connector variables (and proxyServerThread, if needed) if they are currently null. This will be done once per class, but we + * cannot do this with a @BeforeClass because we want to use the type of the subclass to determine if it is a unit or integration test. + */ + @Before + public void prepare() throws TTransportException, AccumuloException, AccumuloSecurityException, InterruptedException { + if (instance == null) { + synchronized (ConnectorBase.class) { + if (instance == null) { + ProxyInstance locInst = null; + Connector locConn = null; + if (IntegrationTest.class.isAssignableFrom(getClass())) { + String host = getString("accumulo.proxy.host"); + int port = getInt("accumulo.proxy.port"); + String user = getString("accumulo.proxy.user"); + String passwd = getString("accumulo.proxy.password"); + locInst = new ProxyInstance(host, port); + locConn = locInst.getConnector(user, new PasswordToken(passwd)); + } else { + createLocalServer(); + locInst = new ProxyInstance("localhost", 45678); + locConn = locInst.getConnector("root", new PasswordToken("")); + } + instance = locInst; + connector = locConn; + } + } + } + } + + @AfterClass + public static void teardown() { + try { + if (instance != null) { + instance.close(); + } + } finally { + instance = null; + } + + try { + if (proxyServerThread != null) { + proxyServerThread.interrupt(); + } + } finally { + proxyServerThread = null; + } + } + + private static String getString(String key) { + String val = System.getProperty(key); + if (val == null) { + Assert.fail("You must specify the system property: " + key); + } + return val.trim(); + } + + private static int getInt(String key) throws NumberFormatException { + return Integer.parseInt(getString(key)); + } + + private static void createLocalServer() throws InterruptedException { + proxyServerThread = new Thread() { + public void run() { + try { + String[] args = new String[] {"-p", "src/test/resources/mock.props"}; + Proxy.main(args); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + }; + proxyServerThread.setDaemon(true); + proxyServerThread.start(); + Thread.sleep(150); + } + +} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/IntegrationTest.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/IntegrationTest.java new file mode 100644 index 0000000..67b8b1e --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/IntegrationTest.java @@ -0,0 +1,24 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +/** + * A flag interface that signals a test is an integration test (vs. a unit test). When used in conjunction with the {@link ConnectorBase} class, the appropriate + * Instance type will be instantiated. + */ +public interface IntegrationTest { + +} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTest.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTest.java new file mode 100644 index 0000000..7255354 --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTest.java @@ -0,0 +1,140 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.charset.StandardCharsets; +import java.util.Map.Entry; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class ScannerTest extends ConnectorBase { + + String table = "__TEST_TABLE__"; + + int noAuthCount; + + @Before + public void createTable() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { + connector.tableOperations().create(table); + BatchWriter bw = connector.createBatchWriter(table, new BatchWriterConfig()); + for (char ch = 'a'; ch <= 'z'; ch++) { + Mutation m = new Mutation(ch + "_row"); + + m.put("fam1", "qual1", "val1:1"); + m.put("fam1", "qual2", "val1:2"); + m.put("fam2", "qual1", "val2:1"); + m.put("fam2", "qual2", "val2:2"); + m.put("fam2", "qual3", "val2:3"); + noAuthCount = m.getUpdates().size(); + bw.addMutation(m); + } + bw.close(); + } + + @After + public void dropTable() throws AccumuloException, AccumuloSecurityException, TableNotFoundException { + connector.tableOperations().delete(table); + } + + @Test + public void simpleTest() throws TableNotFoundException { + Scanner s = connector.createScanner(table, Authorizations.EMPTY); + validate(noAuthCount * 26, s); + } + + @Test + public void testRanges() throws TableNotFoundException { + // first row + Scanner scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("a_row"), noAuthCount, scanner); + + // last row + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("z_row"), noAuthCount, scanner); + + // "random" inner row + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("q_row"), noAuthCount, scanner); + + // some actual ranges + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("d_row", true, "h_row", true), 5 * noAuthCount, scanner); + + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("d_row", false, "h_row", true), 4 * noAuthCount, scanner); + + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("d_row", true, "h_row", false), 4 * noAuthCount, scanner); + + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("d_row", false, "h_row", false), 3 * noAuthCount, scanner); + + // no start + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range(null, "j_row"), 10 * noAuthCount, scanner); + + // no end + scanner = connector.createScanner(table, Authorizations.EMPTY); + testRange(new Range("j_row", null), 17 * noAuthCount, scanner); + + } + + private void testRange(Range range, int expected, Scanner scanner) { + scanner.setRange(range); + Assert.assertEquals(range, scanner.getRange()); + validate(expected, scanner); + } + + private void validate(int expected, Scanner scanner) { + int count = 0; + for (Entry entry : scanner) { + validate(entry); + count++; + } + Assert.assertEquals(expected, count); + scanner.close(); + } + + /** + * expects ("famX", "qualY") -> "valX:Y" + * + * @param entry + * entry to validate + */ + private void validate(Entry entry) { + String fam = entry.getKey().getColumnFamily().toString().substring(3); + String qual = entry.getKey().getColumnQualifier().toString().substring(4); + String val = new String(entry.getValue().get(), StandardCharsets.UTF_8).substring(3); + String[] parts = val.split(":"); + Assert.assertEquals("Family and value did not line up.", fam, parts[0]); + Assert.assertEquals("Qualifier and value did not line up.", qual, parts[1]); + } +} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTestIT.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTestIT.java new file mode 100644 index 0000000..76451f8 --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ScannerTestIT.java @@ -0,0 +1,18 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +public class ScannerTestIT extends ScannerTest implements IntegrationTest {} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTest.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTest.java new file mode 100644 index 0000000..faa1a0b --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTest.java @@ -0,0 +1,78 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.junit.Assert; +import org.junit.Test; + +public class TableOpsTest extends ConnectorBase { + + String table = "__TEST_TABLE__"; + + @Test + public void testTableOps() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { + TableOperations tops = connector.tableOperations(); + + Assert.assertFalse(tops.exists(table)); + tops.create(table); + Assert.assertTrue(tops.exists(table)); + tops.delete(table); + Assert.assertFalse(tops.exists(table)); + } + + @Test + public void testCreateExistingTable() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { + TableOperations tops = connector.tableOperations(); + try { + Assert.assertFalse(tops.exists(table)); + tops.create(table); + Assert.assertTrue(tops.exists(table)); + + try { + tops.create(table); + Assert.fail("Expected second table create to fail."); + } catch (TableExistsException tee) { + // expected + Assert.assertTrue(true); + } + } finally { + if (tops.exists(table)) { + tops.delete(table); + } + Assert.assertFalse(tops.exists(table)); + } + } + + @Test + public void testDeleteNonExistentTable() throws AccumuloException, AccumuloSecurityException { + TableOperations tops = connector.tableOperations(); + + Assert.assertFalse(tops.exists(table)); + try { + tops.delete(table); + Assert.fail("Expected second table create to fail."); + } catch (TableNotFoundException tnfe) { + // expected + Assert.assertTrue(true); + } + } + +} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTestIT.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTestIT.java new file mode 100644 index 0000000..30e4488 --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/TableOpsTestIT.java @@ -0,0 +1,18 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +public class TableOpsTestIT extends TableOpsTest implements IntegrationTest {} diff --git a/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ThiftHelperTest.java b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ThiftHelperTest.java new file mode 100644 index 0000000..83cac7c --- /dev/null +++ b/proxy-instance/src/test/java/edu/jhuapl/accumulo/proxy/ThiftHelperTest.java @@ -0,0 +1,129 @@ +/** + * Copyright 2014-2015 The Johns Hopkins University / Applied Physics Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.jhuapl.accumulo.proxy; + +import java.nio.ByteBuffer; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.hadoop.io.Text; +import org.junit.Assert; +import org.junit.Test; + +public class ThiftHelperTest { + + /** + * Used for enum mapping testing + */ + public enum FirstEnum { + val1, val2 + }; + + /** + * Used for enum mapping testing + */ + public enum SecondEnum { + val1 + }; + + @Test + public void testToText() { + String test = "test"; + + Text txt = ThriftHelper.toText(test); + Assert.assertEquals(test, txt.toString()); + + txt = ThriftHelper.toText(test.getBytes()); + Assert.assertEquals(test, txt.toString()); + + txt = ThriftHelper.toText((byte[]) null); + Assert.assertNull(txt); + } + + @Test + public void testEnum() { + SecondEnum se = ThriftHelper.convertEnum(null, SecondEnum.class); + Assert.assertNull(se); + + se = ThriftHelper.convertEnum(FirstEnum.val1, SecondEnum.class); + Assert.assertEquals(SecondEnum.val1, se); + } + + @Test(expected = ProxyInstanceError.class) + public void testEnumBadMapping() { + ThriftHelper.convertEnum(FirstEnum.val2, SecondEnum.class); + } + + @Test + public void testToByteBuffer() { + ByteBuffer buf = ThriftHelper.toByteBuffer(null); + Assert.assertNull(buf); + + Text text = new Text("test test"); + buf = ThriftHelper.toByteBuffer(text); + Assert.assertArrayEquals(text.getBytes(), buf.array()); + } + + @Test + public void testRanges() { + testConvertRange(new Range("a", "b")); + testConvertRange(new Range(null, "b")); + testConvertRange(new Range("a", null)); + testConvertRange(null); + testConvertRange(new Range("a", true, "b", false)); + testConvertRange(new Range("a", false, "b", false)); + testConvertRange(new Range("a", false, "b", true)); + } + + private void testConvertRange(Range range) { + org.apache.accumulo.proxy.thrift.Range r2 = ThriftHelper.toThrift(range); + checkRanges(range, r2); + + range = ThriftHelper.fromThrift(r2); + checkRanges(range, r2); + } + + private void checkRanges(Range r1, org.apache.accumulo.proxy.thrift.Range r2) { + if (r1 == null) { + Assert.assertNull("t1 was null, but r2 was not!", r2); + return; + } else if (r2 == null) { + Assert.fail("r1 was not null, but r2 was!"); + return; + } + + checkKeys(r1.getStartKey(), r2.getStart()); + checkKeys(r1.getEndKey(), r2.getStop()); + Assert.assertEquals(r1.isStartKeyInclusive(), r2.isStartInclusive()); + Assert.assertEquals(r1.isEndKeyInclusive(), r2.isStopInclusive()); + } + + private void checkKeys(Key k, org.apache.accumulo.proxy.thrift.Key k2) { + if (k == null) { + Assert.assertNull("k was null, but k2 was not!", k2); + return; + } else if (k2 == null) { + Assert.fail("k was not null, but k2 was!"); + return; + } + Assert.assertArrayEquals(k.getRow().getBytes(), k2.getRow()); + Assert.assertArrayEquals(k.getColumnFamily().getBytes(), k2.getColFamily()); + Assert.assertArrayEquals(k.getColumnQualifier().getBytes(), k2.getColQualifier()); + Assert.assertArrayEquals(k.getColumnVisibility().getBytes(), k2.getColVisibility()); + Assert.assertEquals(k.getTimestamp(), k2.getTimestamp()); + } + +} diff --git a/proxy-instance/src/test/resources/mock.props b/proxy-instance/src/test/resources/mock.props new file mode 100644 index 0000000..b0822f8 --- /dev/null +++ b/proxy-instance/src/test/resources/mock.props @@ -0,0 +1,3 @@ +useMockInstance=true +port=45678 +